From d3785d822d4c9bc0e08f27f05e041bef217c11f8 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Fri, 17 Nov 2017 17:01:19 -0800 Subject: [PATCH 1/5] [tf][cli] support cross_account_ids in cloudtrail module --- docs/source/clusters.rst | 22 ++++++++++--------- stream_alert_cli/terraform/cloudtrail.py | 5 ++++- .../tf_stream_alert_cloudtrail/main.tf | 3 +-- .../tf_stream_alert_cloudtrail/variables.tf | 4 ++-- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/source/clusters.rst b/docs/source/clusters.rst index 3b782a836..5df45dae3 100644 --- a/docs/source/clusters.rst +++ b/docs/source/clusters.rst @@ -262,21 +262,23 @@ By default, all API calls will be logged and accessible from rules. { "cloudtrail": { - "enable_logging": true + "enable_logging": true, + "enable_kinesis": true } } **Options:** -=================== ======== ================================== =========== -Key Required Default Description -------------------- -------- ---------------------------------- ----------- -``enable_logging`` ``Yes`` Enable/disable the CloudTrail logging. -``enable_kinesis`` ``No`` ``true`` Enable/disable the sending CloudTrail data to Kinesis. -``existing_trail`` ``No`` ``false`` Set to ``true`` if the account has an existing CloudTrail. This is to avoid duplication of data collected by multiple CloudTrails. -``is_global_trail`` ``No`` ``true`` If the CloudTrail should collect events from any region. -``event_pattern`` ``No`` ``{"account": [""]}`` The CloudWatch Events pattern to send to Kinesis. `More information `_. -=================== ======== ================================== =========== +===================== ======== ================================== =========== +Key Required Default Description +--------------------- -------- ---------------------------------- ----------- +``enable_logging`` ``Yes`` Enable/disable the CloudTrail logging. +``enable_kinesis`` ``No`` ``true`` Enable/disable the sending CloudTrail data to Kinesis. +``existing_trail`` ``No`` ``false`` Set to ``true`` if the account has an existing CloudTrail. This is to avoid duplication of data collected by multiple CloudTrails. +``is_global_trail`` ``No`` ``true`` If the CloudTrail should collect events from any region. +``event_pattern`` ``No`` ``{"account": [""]}`` The CloudWatch Events pattern to send to Kinesis. `More information `_. +``cross_account_ids`` ``No`` Account IDs to grant write access to the created CloudTrail S3 bucket +===================== ======== ================================== =========== Module: Flow Logs ----------------- diff --git a/stream_alert_cli/terraform/cloudtrail.py b/stream_alert_cli/terraform/cloudtrail.py index 84b5e388a..143d9779b 100644 --- a/stream_alert_cli/terraform/cloudtrail.py +++ b/stream_alert_cli/terraform/cloudtrail.py @@ -35,6 +35,9 @@ def generate_cloudtrail(cluster_name, cluster_dict, config): enabled_legacy = modules['cloudtrail'].get('enabled') cloudtrail_enabled = modules['cloudtrail'].get('enable_logging') kinesis_enabled = modules['cloudtrail'].get('enable_kinesis') + account_ids = list(set( + [config['global']['account']['aws_account_id']] + + modules['cloudtrail'].get('cross_account_ids', []))) # Allow for backwards compatilibity if enabled_legacy: @@ -61,7 +64,7 @@ def generate_cloudtrail(cluster_name, cluster_dict, config): cluster_dict['module'][cloudtrail_module] = { 'source': 'modules/tf_stream_alert_cloudtrail', - 'account_id': config['global']['account']['aws_account_id'], + 'account_ids': account_ids, 'cluster': cluster_name, 'prefix': config['global']['account']['prefix'], 'enable_logging': cloudtrail_enabled, diff --git a/terraform/modules/tf_stream_alert_cloudtrail/main.tf b/terraform/modules/tf_stream_alert_cloudtrail/main.tf index f50eb31a1..ec8337a32 100644 --- a/terraform/modules/tf_stream_alert_cloudtrail/main.tf +++ b/terraform/modules/tf_stream_alert_cloudtrail/main.tf @@ -2,7 +2,6 @@ resource "aws_cloudtrail" "streamalert" { name = "${var.prefix}.${var.cluster}.streamalert.cloudtrail" s3_bucket_name = "${aws_s3_bucket.cloudtrail_bucket.id}" - s3_key_prefix = "cloudtrail" enable_log_file_validation = true enable_logging = "${var.enable_logging}" include_global_service_events = true @@ -59,7 +58,7 @@ data "aws_iam_policy_document" "cloudtrail_bucket" { ] resources = [ - "arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail/*", + "${formatlist("arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail/AWSLogs/%s/*", var.account_ids)}", ] principals { diff --git a/terraform/modules/tf_stream_alert_cloudtrail/variables.tf b/terraform/modules/tf_stream_alert_cloudtrail/variables.tf index 80ec1474c..98a3ec0dc 100644 --- a/terraform/modules/tf_stream_alert_cloudtrail/variables.tf +++ b/terraform/modules/tf_stream_alert_cloudtrail/variables.tf @@ -1,5 +1,5 @@ -variable "account_id" { - type = "string" +variable "account_ids" { + type = "list" } variable "cluster" { From d831f29bbfcd24cb4398e0c971a857e6064f2446 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Fri, 17 Nov 2017 17:18:35 -0800 Subject: [PATCH 2/5] [tf][modules]fix bug in Lambda S3 policy naming --- terraform/modules/tf_stream_alert_s3_events/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/tf_stream_alert_s3_events/main.tf b/terraform/modules/tf_stream_alert_s3_events/main.tf index f1c3c4ee0..30a673bdc 100644 --- a/terraform/modules/tf_stream_alert_s3_events/main.tf +++ b/terraform/modules/tf_stream_alert_s3_events/main.tf @@ -21,7 +21,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" { } resource "aws_iam_role_policy" "lambda_s3_permission" { - name = "InvokeFromS3Bucket${title(replace(var.bucket_id, ".", ""))}" + name = "S3GetObjectsFrom${title(replace(var.bucket_id, ".", ""))}" role = "${var.lambda_role_id}" policy = "${data.aws_iam_policy_document.s3_read_only.json}" From c9c217ad0ee557a3ad2f80b447ba0dd020bd7a51 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Fri, 17 Nov 2017 17:19:13 -0800 Subject: [PATCH 3/5] [cli][tf] explicitly add enabled alarms in monitoring module --- stream_alert_cli/terraform/monitoring.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/stream_alert_cli/terraform/monitoring.py b/stream_alert_cli/terraform/monitoring.py index c01e74821..3f106a072 100644 --- a/stream_alert_cli/terraform/monitoring.py +++ b/stream_alert_cli/terraform/monitoring.py @@ -49,18 +49,24 @@ def generate_monitoring(cluster_name, cluster_dict, config): cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)] = { 'source': 'modules/tf_stream_alert_monitoring', - 'sns_topic_arn': sns_topic_arn + 'sns_topic_arn': sns_topic_arn, + 'kinesis_alarms_enabled': False, + 'lambda_alarms_enabled': False } if monitoring_config.get('lambda_alarms_enabled', True): - cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)][ - 'lambda_functions'] = [ + cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)].update({ + 'lambda_functions': [ '{}_{}_streamalert_rule_processor'.format(prefix, cluster_name), '{}_{}_streamalert_alert_processor'.format(prefix, cluster_name) - ] + ], + 'lambda_alarms_enabled': True + }) if monitoring_config.get('kinesis_alarms_enabled', True): - cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)][ - 'kinesis_stream'] = '{}_{}_stream_alert_kinesis'.format(prefix, cluster_name) + cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)].update({ + 'kinesis_stream': '{}_{}_stream_alert_kinesis'.format(prefix, cluster_name), + 'kinesis_alarms_enabled': True + }) return True From b265d102baf9835d153cad970ea4dd1ced37d152 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Fri, 17 Nov 2017 17:20:14 -0800 Subject: [PATCH 4/5] [cli] kinesis config sub parser and handler --- manage.py | 107 +++++++++++++++++---------- stream_alert_cli/kinesis/__init__.py | 0 stream_alert_cli/kinesis/handler.py | 56 ++++++++++++++ stream_alert_cli/runner.py | 4 + 4 files changed, 126 insertions(+), 41 deletions(-) create mode 100644 stream_alert_cli/kinesis/__init__.py create mode 100644 stream_alert_cli/kinesis/handler.py diff --git a/manage.py b/manage.py index 4dff36307..b79f103bf 100755 --- a/manage.py +++ b/manage.py @@ -36,6 +36,12 @@ from app_integrations.apps.app_base import StreamAlertApp +CLUSTERS = [ + os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters') + for cluster in files +] + + class UniqueSetAction(Action): """Subclass of argparse.Action to avoid multiple of the same choice from a list""" @@ -139,15 +145,9 @@ def _add_live_test_subparser(subparsers): # set the name of this parser to 'live-test' live_test_parser.set_defaults(command='live-test') - # get cluster choices from available files - clusters = [ - os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters') - for cluster in files - ] - # add clusters for user to pick from live_test_parser.add_argument( - '-c', '--cluster', choices=clusters, help=ARGPARSE_SUPPRESS, required=True) + '-c', '--cluster', choices=CLUSTERS, help=ARGPARSE_SUPPRESS, required=True) # add the optional ability to test against a rule/set of rules live_test_parser.add_argument( @@ -230,12 +230,6 @@ def _add_app_integration_subparser(subparsers): formatter_class=RawTextHelpFormatter, help=ARGPARSE_SUPPRESS) - # get cluster choices from available files - clusters = [ - os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters') - for cluster in files - ] - # Set the name of this parser to 'app' app_integration_parser.set_defaults(command='app') @@ -245,9 +239,9 @@ def _add_app_integration_subparser(subparsers): _add_app_integration_new_subparser( app_integration_subparsers, sorted(StreamAlertApp.get_all_apps()), - clusters + CLUSTERS ) - _add_app_integration_update_auth_subparser(app_integration_subparsers, clusters) + _add_app_integration_update_auth_subparser(app_integration_subparsers, CLUSTERS) def _add_app_integration_list_subparser(subparsers): @@ -487,13 +481,7 @@ def _add_metrics_subparser(subparsers): """Add the metrics subparser: manage.py metrics [options]""" metrics_usage = 'manage.py metrics [options]' - # get cluster choices from available files - clusters = [ - os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters') - for cluster in files - ] - - cluster_choices_block = ('\n').join('{:>28}{}'.format('', cluster) for cluster in clusters) + cluster_choices_block = ('\n').join('{:>28}{}'.format('', cluster) for cluster in CLUSTERS) metrics_description = (""" StreamAlertCLI v{} @@ -552,11 +540,11 @@ def _add_metrics_subparser(subparsers): metrics_parser.add_argument( '-c', '--clusters', - choices=clusters, + choices=CLUSTERS, help=ARGPARSE_SUPPRESS, nargs='+', action=UniqueSetAction, - default=clusters) + default=CLUSTERS) # allow verbose output for the CLI with the --debug option metrics_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS) @@ -572,13 +560,7 @@ def _add_metric_alarm_subparser(subparsers): metric_choices_block = ('\n').join('{:>35}{}'.format('', metric) for metric in all_metrics) - # get cluster choices from available files - clusters = [ - os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters') - for cluster in files - ] - - cluster_choices_block = ('\n').join('{:>37}{}'.format('', cluster) for cluster in clusters) + cluster_choices_block = ('\n').join('{:>37}{}'.format('', cluster) for cluster in CLUSTERS) metric_alarm_description = (""" StreamAlertCLI v{} @@ -758,7 +740,7 @@ def _alarm_description_validator(val): metric_alarm_parser.add_argument( '-c', '--clusters', - choices=clusters, + choices=CLUSTERS, help=ARGPARSE_SUPPRESS, nargs='+', action=UniqueSetAction, @@ -1067,6 +1049,55 @@ def _add_terraform_subparser(subparsers): tf_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS) +def _add_kinesis_subparser(subparsers): + """Add kinesis subparser""" + kinesis_usage = 'manage.py kinesis [disable-events]' + kinesis_description = (""" +StreamAlertCLI v{} +Kinesis StreamAlert options + +Update Kinesis settings and then runs Terraform + +Available Commands: + + disable-events Disable Kinesis Events + enable-events Enable Kinesis Events + +Arguments: + + --clusters Space delimited set of clusters to modify, defaults to all + --debug Debug mode + --skip-terraform Only set the config, do not run Terraform after + +Examples: + + manage.py kinesis disable-events --clusters corp prod + +""".format(version)) + kinesis_parser = subparsers.add_parser( + 'kinesis', + usage=kinesis_usage, + description=kinesis_description, + help=ARGPARSE_SUPPRESS, + formatter_class=RawTextHelpFormatter) + + kinesis_parser.set_defaults(command='kinesis') + kinesis_parser.add_argument( + 'subcommand', + choices=['disable-events', 'enable-events'], + help=ARGPARSE_SUPPRESS) + kinesis_parser.add_argument( + '-c', + '--clusters', + choices=CLUSTERS, + help=ARGPARSE_SUPPRESS, + nargs='+', + action=UniqueSetAction, + default=set()) + kinesis_parser.add_argument('--skip-terraform', action='store_true', help=ARGPARSE_SUPPRESS) + kinesis_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS) + + def _add_configure_subparser(subparsers): """Add configure subparser: manage.py configure [config_key] [config_value]""" configure_usage = 'manage.py configure [config_key] [config_value]' @@ -1119,16 +1150,9 @@ def _add_athena_subparser(subparsers): manage.py athena create-db - manage.py athena create-table \ - --type alerts \ - --bucket s3.bucket.name \ - --refresh_type add_hive_partition + manage.py athena create-table --type alerts --bucket s3.bucket.name --refresh_type add_hive_partition - manage.py athena create-table \ - --type data \ - --bucket s3.bucket.name \ - --refresh_type add_hive_partition \ - --table_name my_athena_table + manage.py athena create-table --type data --bucket s3.bucket.name --refresh_type add_hive_partition --table_name my_athena_table """.format(version)) athena_parser = subparsers.add_parser( @@ -1198,6 +1222,7 @@ def build_parser(): _add_configure_subparser(subparsers) _add_athena_subparser(subparsers) _add_app_integration_subparser(subparsers) + _add_kinesis_subparser(subparsers) return parser diff --git a/stream_alert_cli/kinesis/__init__.py b/stream_alert_cli/kinesis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stream_alert_cli/kinesis/handler.py b/stream_alert_cli/kinesis/handler.py new file mode 100644 index 000000000..d9e7e9a5b --- /dev/null +++ b/stream_alert_cli/kinesis/handler.py @@ -0,0 +1,56 @@ +""" +Copyright 2017-present, Airbnb Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from stream_alert_cli.helpers import tf_runner +from stream_alert_cli.logger import LOGGER_CLI +from stream_alert_cli.terraform.generate import terraform_generate + + +def kinesis_handler(options, config): + """Main handler for the Kinesis parser + + Args: + options (namedtuple): Parsed arguments + config (CLIConfig): Loaded StreamAlert config + """ + if options.subcommand == 'disable-events': + LOGGER_CLI.info('Disabling Kinesis Events') + set_kinesis_events(options, config, False) + elif options.subcommand == 'enable-events': + LOGGER_CLI.info('Enabling Kinesis Events') + set_kinesis_events(options, config, True) + + +def set_kinesis_events(options, config, enable=True): + """Enable or disable Kinesis events for given clusters + + Args: + options (namedtuple): Parsed arguments + config (CLIConfig): Loaded StreamAlert config + enable (bool): Enable/Disable switch + """ + for cluster in options.clusters or config.clusters(): + if 'kinesis_events' in config['clusters'][cluster]['modules']: + config['clusters'][cluster]['modules']['kinesis_events']['enabled'] = enable + + config.write() + + if not options.skip_terraform: + terraform_generate(config) + tf_runner( + action='apply', + targets=[ + 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters() + ]) diff --git a/stream_alert_cli/runner.py b/stream_alert_cli/runner.py index 6d0e5d5f7..d50f842e0 100644 --- a/stream_alert_cli/runner.py +++ b/stream_alert_cli/runner.py @@ -19,6 +19,7 @@ from stream_alert_cli.athena.handler import athena_handler from stream_alert_cli.config import CLIConfig from stream_alert_cli.helpers import user_input +from stream_alert_cli.kinesis.handler import kinesis_handler from stream_alert_cli.logger import LOGGER_CLI from stream_alert_cli.manage_lambda.handler import lambda_handler from stream_alert_cli.terraform.handler import terraform_handler @@ -74,6 +75,9 @@ def cli_runner(options): elif options.command == 'app': _app_integration_handler(options) + elif options.command == 'kinesis': + kinesis_handler(options, CONFIG) + def configure_handler(options): """Configure StreamAlert main settings From 03d254e4f9adf9c10161c31d91294a6157cc2e17 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Mon, 20 Nov 2017 13:33:46 -0800 Subject: [PATCH 5/5] [tests] unit test cloudtrail/cloudwatch tf module changes --- .../stream_alert_cli/terraform/test_generate.py | 4 ++-- .../terraform/test_monitoring.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/unit/stream_alert_cli/terraform/test_generate.py b/tests/unit/stream_alert_cli/terraform/test_generate.py index 4f3ad2251..224d1813f 100644 --- a/tests/unit/stream_alert_cli/terraform/test_generate.py +++ b/tests/unit/stream_alert_cli/terraform/test_generate.py @@ -326,7 +326,7 @@ def test_generate_cloudtrail_basic(self): assert_equal(set(self.config['clusters']['advanced']['modules']['cloudtrail'].keys()), {'enable_logging', 'enable_kinesis'}) assert_equal(self.cluster_dict['module']['cloudtrail_advanced'], { - 'account_id': '12345678910', + 'account_ids': ['12345678910'], 'cluster': 'advanced', 'kinesis_arn': '${module.kinesis_advanced.arn}', 'prefix': 'unit-testing', @@ -363,7 +363,7 @@ def test_generate_cloudtrail_all_options(self): assert_equal('cloudtrail_advanced' in self.cluster_dict['module'], True) assert_equal(self.cluster_dict['module']['cloudtrail_advanced'], { - 'account_id': '12345678910', + 'account_ids': ['12345678910'], 'cluster': 'advanced', 'existing_trail': False, 'is_global_trail': False, diff --git a/tests/unit/stream_alert_cli/terraform/test_monitoring.py b/tests/unit/stream_alert_cli/terraform/test_monitoring.py index 77675d46c..ad6212313 100644 --- a/tests/unit/stream_alert_cli/terraform/test_monitoring.py +++ b/tests/unit/stream_alert_cli/terraform/test_monitoring.py @@ -34,7 +34,9 @@ def test_generate_cloudwatch_monitoring(): 'unit-testing_test_streamalert_rule_processor', 'unit-testing_test_streamalert_alert_processor' ], - 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis' + 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis', + 'lambda_alarms_enabled': True, + 'kinesis_alarms_enabled': True } assert_true(result) @@ -56,7 +58,9 @@ def test_generate_cloudwatch_monitoring_no_kinesis(): 'lambda_functions': [ 'unit-testing_test_streamalert_rule_processor', 'unit-testing_test_streamalert_alert_processor' - ] + ], + 'lambda_alarms_enabled': True, + 'kinesis_alarms_enabled': False } assert_true(result) @@ -75,7 +79,9 @@ def test_generate_cloudwatch_monitoring_no_lambda(): expected_cloudwatch_tf = { 'source': 'modules/tf_stream_alert_monitoring', 'sns_topic_arn': 'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring', - 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis' + 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis', + 'lambda_alarms_enabled': False, + 'kinesis_alarms_enabled': True } assert_true(result) @@ -102,7 +108,9 @@ def test_generate_cloudwatch_monitoring_custom_sns(): 'unit-testing_test_streamalert_rule_processor', 'unit-testing_test_streamalert_alert_processor' ], - 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis' + 'kinesis_stream': 'unit-testing_test_stream_alert_kinesis', + 'lambda_alarms_enabled': True, + 'kinesis_alarms_enabled': True } assert_true(result)