Cross Account CloudTrail Support, Kinesis CLI Parser

docs/source/clusters.rst

@@ -262,21 +262,23 @@ By default, all API calls will be logged and accessible from rules.
 
   {
     "cloudtrail": {
-      "enable_logging": true
+      "enable_logging": true,
+      "enable_kinesis": true
     }
   }
 
 **Options:**
 
-===================  ========  ==================================  ===========
-Key                  Required  Default                             Description
--------------------  --------  ----------------------------------  -----------
-``enable_logging``   ``Yes``                                       Enable/disable the CloudTrail logging.
-``enable_kinesis``   ``No``    ``true``                            Enable/disable the sending CloudTrail data to Kinesis.
-``existing_trail``   ``No``    ``false``                           Set to ``true`` if the account has an existing CloudTrail.  This is to avoid duplication of data collected by multiple CloudTrails.
-``is_global_trail``  ``No``    ``true``                            If the CloudTrail should collect events from any region.
-``event_pattern``    ``No``    ``{"account": ["<accound_id>"]}``   The CloudWatch Events pattern to send to Kinesis.  `More information <http://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html>`_.
-===================  ========  ==================================  ===========
+=====================    ========  ==================================  ===========
+Key                      Required  Default                             Description
+---------------------    --------  ----------------------------------  -----------
+``enable_logging``       ``Yes``                                       Enable/disable the CloudTrail logging.
+``enable_kinesis``       ``No``    ``true``                            Enable/disable the sending CloudTrail data to Kinesis.
+``existing_trail``       ``No``    ``false``                           Set to ``true`` if the account has an existing CloudTrail.  This is to avoid duplication of data collected by multiple CloudTrails.
+``is_global_trail``      ``No``    ``true``                            If the CloudTrail should collect events from any region.
+``event_pattern``        ``No``    ``{"account": ["<accound_id>"]}``   The CloudWatch Events pattern to send to Kinesis.  `More information <http://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html>`_.
+``cross_account_ids``    ``No``                                        Account IDs to grant write access to the created CloudTrail S3 bucket
+=====================    ========  ==================================  ===========
 
 Module: Flow Logs
 -----------------

manage.py

@@ -36,6 +36,12 @@
 from app_integrations.apps.app_base import StreamAlertApp
 
 
+CLUSTERS = [
+    os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters')
+    for cluster in files
+]
+
+
 class UniqueSetAction(Action):
     """Subclass of argparse.Action to avoid multiple of the same choice from a list"""
 
@@ -139,15 +145,9 @@ def _add_live_test_subparser(subparsers):
     # set the name of this parser to 'live-test'
     live_test_parser.set_defaults(command='live-test')
 
-    # get cluster choices from available files
-    clusters = [
-        os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters')
-        for cluster in files
-    ]
-
     # add clusters for user to pick from
     live_test_parser.add_argument(
-        '-c', '--cluster', choices=clusters, help=ARGPARSE_SUPPRESS, required=True)
+        '-c', '--cluster', choices=CLUSTERS, help=ARGPARSE_SUPPRESS, required=True)
 
     # add the optional ability to test against a rule/set of rules
     live_test_parser.add_argument(
@@ -230,12 +230,6 @@ def _add_app_integration_subparser(subparsers):
         formatter_class=RawTextHelpFormatter,
         help=ARGPARSE_SUPPRESS)
 
-    # get cluster choices from available files
-    clusters = [
-        os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters')
-        for cluster in files
-    ]
-
     # Set the name of this parser to 'app'
     app_integration_parser.set_defaults(command='app')
 
@@ -245,9 +239,9 @@ def _add_app_integration_subparser(subparsers):
     _add_app_integration_new_subparser(
         app_integration_subparsers,
         sorted(StreamAlertApp.get_all_apps()),
-        clusters
+        CLUSTERS
     )
-    _add_app_integration_update_auth_subparser(app_integration_subparsers, clusters)
+    _add_app_integration_update_auth_subparser(app_integration_subparsers, CLUSTERS)
 
 
 def _add_app_integration_list_subparser(subparsers):
@@ -487,13 +481,7 @@ def _add_metrics_subparser(subparsers):
     """Add the metrics subparser: manage.py metrics [options]"""
     metrics_usage = 'manage.py metrics [options]'
 
-    # get cluster choices from available files
-    clusters = [
-        os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters')
-        for cluster in files
-    ]
-
-    cluster_choices_block = ('\n').join('{:>28}{}'.format('', cluster) for cluster in clusters)
+    cluster_choices_block = ('\n').join('{:>28}{}'.format('', cluster) for cluster in CLUSTERS)
 
     metrics_description = ("""
 StreamAlertCLI v{}
@@ -552,11 +540,11 @@ def _add_metrics_subparser(subparsers):
     metrics_parser.add_argument(
         '-c',
         '--clusters',
-        choices=clusters,
+        choices=CLUSTERS,
         help=ARGPARSE_SUPPRESS,
         nargs='+',
         action=UniqueSetAction,
-        default=clusters)
+        default=CLUSTERS)
 
     # allow verbose output for the CLI with the --debug option
     metrics_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS)
@@ -572,13 +560,7 @@ def _add_metric_alarm_subparser(subparsers):
 
     metric_choices_block = ('\n').join('{:>35}{}'.format('', metric) for metric in all_metrics)
 
-    # get cluster choices from available files
-    clusters = [
-        os.path.splitext(cluster)[0] for _, _, files in os.walk('conf/clusters')
-        for cluster in files
-    ]
-
-    cluster_choices_block = ('\n').join('{:>37}{}'.format('', cluster) for cluster in clusters)
+    cluster_choices_block = ('\n').join('{:>37}{}'.format('', cluster) for cluster in CLUSTERS)
 
     metric_alarm_description = ("""
 StreamAlertCLI v{}
@@ -758,7 +740,7 @@ def _alarm_description_validator(val):
     metric_alarm_parser.add_argument(
         '-c',
         '--clusters',
-        choices=clusters,
+        choices=CLUSTERS,
         help=ARGPARSE_SUPPRESS,
         nargs='+',
         action=UniqueSetAction,
@@ -1067,6 +1049,55 @@ def _add_terraform_subparser(subparsers):
     tf_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS)
 
 
+def _add_kinesis_subparser(subparsers):
+    """Add kinesis subparser"""
+    kinesis_usage = 'manage.py kinesis [disable-events]'
+    kinesis_description = ("""
+StreamAlertCLI v{}
+Kinesis StreamAlert options
+
+Update Kinesis settings and then runs Terraform
+
+Available Commands:
+
+    disable-events             Disable Kinesis Events
+    enable-events              Enable Kinesis Events
+
+Arguments:
+
+    --clusters                Space delimited set of clusters to modify, defaults to all
+    --debug                   Debug mode
+    --skip-terraform          Only set the config, do not run Terraform after                
+
+Examples:
+
+    manage.py kinesis disable-events --clusters corp prod
+
+""".format(version))
+    kinesis_parser = subparsers.add_parser(
+        'kinesis',
+        usage=kinesis_usage,
+        description=kinesis_description,
+        help=ARGPARSE_SUPPRESS,
+        formatter_class=RawTextHelpFormatter)
+
+    kinesis_parser.set_defaults(command='kinesis')
+    kinesis_parser.add_argument(
+        'subcommand',
+        choices=['disable-events', 'enable-events'],
+        help=ARGPARSE_SUPPRESS)
+    kinesis_parser.add_argument(
+        '-c',
+        '--clusters',
+        choices=CLUSTERS,
+        help=ARGPARSE_SUPPRESS,
+        nargs='+',
+        action=UniqueSetAction,
+        default=set())
+    kinesis_parser.add_argument('--skip-terraform', action='store_true', help=ARGPARSE_SUPPRESS)
+    kinesis_parser.add_argument('--debug', action='store_true', help=ARGPARSE_SUPPRESS)
+
+
 def _add_configure_subparser(subparsers):
     """Add configure subparser: manage.py configure [config_key] [config_value]"""
     configure_usage = 'manage.py configure [config_key] [config_value]'
@@ -1119,16 +1150,9 @@ def _add_athena_subparser(subparsers):
 
     manage.py athena create-db
 
-    manage.py athena create-table \
-    --type alerts \
-    --bucket s3.bucket.name \
-    --refresh_type add_hive_partition
+    manage.py athena create-table --type alerts --bucket s3.bucket.name --refresh_type add_hive_partition
 
-    manage.py athena create-table \
-    --type data \
-    --bucket s3.bucket.name \
-    --refresh_type add_hive_partition \
-    --table_name my_athena_table
+    manage.py athena create-table --type data --bucket s3.bucket.name --refresh_type add_hive_partition --table_name my_athena_table
 
 """.format(version))
     athena_parser = subparsers.add_parser(
@@ -1198,6 +1222,7 @@ def build_parser():
     _add_configure_subparser(subparsers)
     _add_athena_subparser(subparsers)
     _add_app_integration_subparser(subparsers)
+    _add_kinesis_subparser(subparsers)
 
     return parser
 

stream_alert_cli/kinesis/handler.py

@@ -0,0 +1,56 @@
+"""
+Copyright 2017-present, Airbnb Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from stream_alert_cli.helpers import tf_runner
+from stream_alert_cli.logger import LOGGER_CLI
+from stream_alert_cli.terraform.generate import terraform_generate
+
+
+def kinesis_handler(options, config):
+    """Main handler for the Kinesis parser
+
+    Args:
+        options (namedtuple): Parsed arguments
+        config (CLIConfig): Loaded StreamAlert config
+    """
+    if options.subcommand == 'disable-events':
+        LOGGER_CLI.info('Disabling Kinesis Events')
+        set_kinesis_events(options, config, False)
+    elif options.subcommand == 'enable-events':
+        LOGGER_CLI.info('Enabling Kinesis Events')
+        set_kinesis_events(options, config, True)
+
+
+def set_kinesis_events(options, config, enable=True):
+    """Enable or disable Kinesis events for given clusters
+
+    Args:
+        options (namedtuple): Parsed arguments
+        config (CLIConfig): Loaded StreamAlert config
+        enable (bool): Enable/Disable switch
+    """
+    for cluster in options.clusters or config.clusters():
+        if 'kinesis_events' in config['clusters'][cluster]['modules']:
+            config['clusters'][cluster]['modules']['kinesis_events']['enabled'] = enable
+
+    config.write()
+
+    if not options.skip_terraform:
+        terraform_generate(config)
+        tf_runner(
+            action='apply',
+            targets=[
+                'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters()
+            ])

stream_alert_cli/runner.py

@@ -19,6 +19,7 @@
 from stream_alert_cli.athena.handler import athena_handler
 from stream_alert_cli.config import CLIConfig
 from stream_alert_cli.helpers import user_input
+from stream_alert_cli.kinesis.handler import kinesis_handler
 from stream_alert_cli.logger import LOGGER_CLI
 from stream_alert_cli.manage_lambda.handler import lambda_handler
 from stream_alert_cli.terraform.handler import terraform_handler
@@ -74,6 +75,9 @@ def cli_runner(options):
     elif options.command == 'app':
         _app_integration_handler(options)
 
+    elif options.command == 'kinesis':
+        kinesis_handler(options, CONFIG)
+
 
 def configure_handler(options):
     """Configure StreamAlert main settings

stream_alert_cli/terraform/cloudtrail.py

@@ -35,6 +35,9 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
     enabled_legacy = modules['cloudtrail'].get('enabled')
     cloudtrail_enabled = modules['cloudtrail'].get('enable_logging')
     kinesis_enabled = modules['cloudtrail'].get('enable_kinesis')
+    account_ids = list(set(
+        [config['global']['account']['aws_account_id']] +
+        modules['cloudtrail'].get('cross_account_ids', [])))
 
     # Allow for backwards compatilibity
     if enabled_legacy:
@@ -61,7 +64,7 @@ def generate_cloudtrail(cluster_name, cluster_dict, config):
 
     cluster_dict['module'][cloudtrail_module] = {
         'source': 'modules/tf_stream_alert_cloudtrail',
-        'account_id': config['global']['account']['aws_account_id'],
+        'account_ids': account_ids,
         'cluster': cluster_name,
         'prefix': config['global']['account']['prefix'],
         'enable_logging': cloudtrail_enabled,

stream_alert_cli/terraform/monitoring.py

@@ -49,18 +49,24 @@ def generate_monitoring(cluster_name, cluster_dict, config):
 
     cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)] = {
         'source': 'modules/tf_stream_alert_monitoring',
-        'sns_topic_arn': sns_topic_arn
+        'sns_topic_arn': sns_topic_arn,
+        'kinesis_alarms_enabled': False,
+        'lambda_alarms_enabled': False
     }
 
     if monitoring_config.get('lambda_alarms_enabled', True):
-        cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)][
-            'lambda_functions'] = [
+        cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)].update({
+            'lambda_functions': [
                 '{}_{}_streamalert_rule_processor'.format(prefix, cluster_name),
                 '{}_{}_streamalert_alert_processor'.format(prefix, cluster_name)
-            ]
+            ],
+            'lambda_alarms_enabled': True
+        })
 
     if monitoring_config.get('kinesis_alarms_enabled', True):
-        cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)][
-            'kinesis_stream'] = '{}_{}_stream_alert_kinesis'.format(prefix, cluster_name)
+        cluster_dict['module']['cloudwatch_monitoring_{}'.format(cluster_name)].update({
+            'kinesis_stream': '{}_{}_stream_alert_kinesis'.format(prefix, cluster_name),
+            'kinesis_alarms_enabled': True
+        })
 
     return True

terraform/modules/tf_stream_alert_cloudtrail/main.tf

@@ -2,7 +2,6 @@
 resource "aws_cloudtrail" "streamalert" {
   name                          = "${var.prefix}.${var.cluster}.streamalert.cloudtrail"
   s3_bucket_name                = "${aws_s3_bucket.cloudtrail_bucket.id}"
-  s3_key_prefix                 = "cloudtrail"
   enable_log_file_validation    = true
   enable_logging                = "${var.enable_logging}"
   include_global_service_events = true
@@ -59,7 +58,7 @@ data "aws_iam_policy_document" "cloudtrail_bucket" {
     ]
 
     resources = [
-      "arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail/*",
+      "${formatlist("arn:aws:s3:::${var.prefix}.${var.cluster}.streamalert.cloudtrail/AWSLogs/%s/*", var.account_ids)}",
     ]
 
     principals {

terraform/modules/tf_stream_alert_cloudtrail/variables.tf

@@ -1,5 +1,5 @@
-variable "account_id" {
-  type = "string"
+variable "account_ids" {
+  type = "list"
 }
 
 variable "cluster" {

terraform/modules/tf_stream_alert_s3_events/main.tf

@@ -21,7 +21,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
 }
 
 resource "aws_iam_role_policy" "lambda_s3_permission" {
-  name = "InvokeFromS3Bucket${title(replace(var.bucket_id, ".", ""))}"
+  name = "S3GetObjectsFrom${title(replace(var.bucket_id, ".", ""))}"
   role = "${var.lambda_role_id}"
 
   policy = "${data.aws_iam_policy_document.s3_read_only.json}"