[cli] fixing bug that with sns topic and proper aggregate metric name

* Updating unit test
* Migarting a constant dict to metrics pacakge to be shared
* Various linting

manage.py

@@ -504,7 +504,8 @@ def _alarm_description_validator(val):
     metric_alarm_parser.add_argument(
         '-ad', '--alarm-description',
         help=ARGPARSE_SUPPRESS,
-        type=_alarm_description_validator
+        type=_alarm_description_validator,
+        default=''
     )
 
     # allow the user to select 0 or more clusters to apply this alarm to
@@ -543,7 +544,8 @@ def _alarm_description_validator(val):
     metric_alarm_parser.add_argument(
         '-s', '--statistic',
         choices=['SampleCount', 'Average', 'Sum', 'Minimum', 'Maximum'],
-        help=ARGPARSE_SUPPRESS
+        help=ARGPARSE_SUPPRESS,
+        default=''
     )
 
     # allow verbose output for the CLI with the --debug option

stream_alert/shared/metrics.py

@@ -24,6 +24,12 @@
 
 CLUSTER = os.environ.get('CLUSTER', 'unknown_cluster')
 
+# The FUNC_PREFIXES dict acts as a simple map to a human-readable name
+# Add ATHENA_PARTITION_REFRESH_NAME: 'AthenaPartitionRefresh', to the
+# below when metrics are supported there
+FUNC_PREFIXES = {ALERT_PROCESSOR_NAME: 'AlertProcessor',
+                 RULE_PROCESSOR_NAME: 'RuleProcessor'}
+
 try:
     ENABLE_METRICS = bool(int(os.environ.get('ENABLE_METRICS', 0)))
 except ValueError as err:

stream_alert_cli/config.py

@@ -208,12 +208,20 @@ def _add_metric_alarm_per_cluster(self, alarm_info, function_name):
                     continue
 
             metric_alarms = function_config.get('metric_alarms', {})
-            new_alarms = self._add_metric_alarm_config(alarm_info, metric_alarms)
+
+            # Format the metric name for the cluster based metric
+            # Prepend a prefix for this function and append the cluster name
+            alarm_settings = alarm_info.copy()
+            alarm_settings['metric_name'] = '{}-{}-{}'.format(metrics.FUNC_PREFIXES[function_name],
+                                                              alarm_settings['metric_name'],
+                                                              cluster.upper())
+
+            new_alarms = self._add_metric_alarm_config(alarm_settings, metric_alarms)
             if new_alarms != False:
                 function_config['metric_alarms'] = new_alarms
                 LOGGER_CLI.info('Successfully added \'%s\' metric alarm for the \'%s\' '
                                 'function to \'conf/clusters/%s.json.\'',
-                                alarm_info['alarm_name'], function_name, cluster)
+                                alarm_settings['alarm_name'], function_name, cluster)
 
     def _alarm_exists(self, alarm_name):
         """Check if this alarm name is already used somewhere. CloudWatch alarm
@@ -332,11 +340,16 @@ def add_metric_alarm(self, alarm_info):
             if not metric_alarms:
                 global_config['metric_alarms'][metric_function] = {}
 
-            new_alarms = self._add_metric_alarm_config(alarm_info, metric_alarms)
+            # Format the metric name for the aggregate metric
+            alarm_settings = alarm_info.copy()
+            alarm_settings['metric_name'] = '{}-{}'.format(metrics.FUNC_PREFIXES[metric_function],
+                                                           alarm_info['metric_name'])
+
+            new_alarms = self._add_metric_alarm_config(alarm_settings, metric_alarms)
             if new_alarms != False:
                 global_config['metric_alarms'][metric_function] = new_alarms
                 LOGGER_CLI.info('Successfully added \'%s\' metric alarm to '
-                                '\'conf/global.json.\'', alarm_info['alarm_name'])
+                                '\'conf/global.json.\'', alarm_settings['alarm_name'])
 
         else:
             # Add metric alarms on a per-cluster basis - these are added to the cluster config

stream_alert_cli/terraform_generate.py

@@ -24,10 +24,6 @@
 DEFAULT_SNS_MONITORING_TOPIC = 'stream_alert_monitoring'
 RESTRICTED_CLUSTER_NAMES = ('main', 'athena')
 
-# The FUNC_PREFIXES dict acts as a simple map to a human-readable name
-FUNC_PREFIXES = {metrics.ALERT_PROCESSOR_NAME: 'AlertProcessor',
-                 metrics.RULE_PROCESSOR_NAME: 'RuleProcessor'}
-
 
 class InvalidClusterName(Exception):
     """Exception for invalid cluster names"""
@@ -179,19 +175,27 @@ def generate_main(**kwargs):
     if not global_metrics:
         return main_dict
 
+    topic_name = (DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config
+                  ['monitoring'].get('create_sns_topic') else
+                  infrastructure_config['monitoring'].get('sns_topic_name'))
+
+    sns_topic_arn = 'arn:aws:sns:{region}:{account_id}:{topic}'.format(
+        region=config['global']['account']['region'],
+        account_id=config['global']['account']['aws_account_id'],
+        topic=topic_name
+    )
+
     formatted_alarms = {}
     # Add global metric alarms for the rule and alert processors
-    for func in FUNC_PREFIXES:
+    for func in metrics.FUNC_PREFIXES:
         if not func in global_metrics:
             continue
 
         for name, settings in global_metrics[func].iteritems():
             alarm_info = settings.copy()
             alarm_info['alarm_name'] = name
             alarm_info['namespace'] = 'StreamAlert'
-            alarm_info['alarm_actions'] = ['${{aws_sns_topic.{}.arn}}'.format(
-                DEFAULT_SNS_MONITORING_TOPIC
-            )]
+            alarm_info['alarm_actions'] = [sns_topic_arn]
             # Terraform only allows certain characters in resource names, so strip the name
             acceptable_chars = ''.join([string.digits, string.letters, '_-'])
             name = filter(acceptable_chars.__contains__, name)
@@ -331,7 +335,7 @@ def generate_cloudwatch_metric_filters(cluster_name, cluster_dict, config):
     current_metrics = metrics.MetricLogger.get_available_metrics()
 
     # Add metric filters for the rule and alert processor
-    for func in FUNC_PREFIXES:
+    for func, metric_prefix in metrics.FUNC_PREFIXES.iteritems():
         if func not in current_metrics:
             continue
 
@@ -344,7 +348,6 @@ def generate_cloudwatch_metric_filters(cluster_name, cluster_dict, config):
         if not stream_alert_config[func].get('enable_metrics'):
             continue
 
-        metric_prefix = FUNC_PREFIXES[func]
         filter_pattern_idx, filter_value_idx = 0, 1
 
         # Add filters for the cluster and aggregate
@@ -367,28 +370,22 @@ def generate_cloudwatch_metric_filters(cluster_name, cluster_dict, config):
             ['{}_metric_filters'.format(func)] = filters
 
 
-def _format_metric_alarm(name, alarm_settings, function, cluster=''):
+def _format_metric_alarm(name, alarm_settings):
     """Helper function to format a metric alarm as a comma-separated string
 
     Args:
-        name (str):
-        alarm_info (dict):
-        function (str):
-        cluster (str):
+        name (str): The name of the alarm to create
+        alarm_info (dict): All other settings for this alarm (threshold, etc)
+        function (str): The respective function this alarm is being created for.
+            This is the RuleProcessor or AlertProcessor
+        cluster (str): The cluster that this metric is related to
 
     Returns:
         str: formatted and comma-separated string containing alarm settings
     """
     alarm_info = alarm_settings.copy()
     # The alarm description and name can potentially have commas so remove them
-    alarm_info['alarm_description'] = (alarm_info['alarm_description'].replace(',', '') if
-                                       alarm_info['alarm_description'] else '')
-
-    # Prepend a prefix for this function to the metric name and append a cluster
-    # name if there is one available
-    alarm_info['metric_name'] = '{}-{}'.format(function, alarm_info['metric_name'])
-    if cluster:
-        alarm_info['metric_name'] = '{}-{}'.format(alarm_info['metric_name'], cluster.upper())
+    alarm_info['alarm_description'] = alarm_info['alarm_description'].replace(',', '')
 
     attributes = list(alarm_info)
     attributes.sort()
@@ -414,9 +411,9 @@ def generate_cloudwatch_metric_alarms(cluster_name, cluster_dict, config):
         LOGGER_CLI.error('Invalid config: Make sure you declare global infrastructure options!')
         return
 
-    topic_name = DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config \
-                 ['monitoring'].get('create_sns_topic') else \
-                 infrastructure_config['monitoring'].get('sns_topic_name')
+    topic_name = (DEFAULT_SNS_MONITORING_TOPIC if infrastructure_config
+                  ['monitoring'].get('create_sns_topic') else
+                  infrastructure_config['monitoring'].get('sns_topic_name'))
 
     sns_topic_arn = 'arn:aws:sns:{region}:{account_id}:{topic}'.format(
         region=config['global']['account']['region'],
@@ -431,14 +428,14 @@ def generate_cloudwatch_metric_alarms(cluster_name, cluster_dict, config):
 
     # Add cluster metric alarms for the rule and alert processors
     formatted_alarms = []
-    for func, func_config in stream_alert_config.iteritems():
+    for func_config in stream_alert_config.values():
         if 'metric_alarms' not in func_config:
             continue
 
         metric_alarms = func_config['metric_alarms']
         for name, alarm_info in metric_alarms.iteritems():
             formatted_alarms.append(
-                _format_metric_alarm(name, alarm_info, FUNC_PREFIXES[func], cluster_name)
+                _format_metric_alarm(name, alarm_info)
             )
 
     cluster_dict['module']['stream_alert_{}'.format(cluster_name)] \

tests/unit/stream_alert_cli/test_terraform_generate.py

@@ -585,14 +585,15 @@ def test_generate_athena(self):
                     'prefix': 'unit-testing'
                 },
                 'infrastructure': {
-                    'metrics': {
-                        'enabled': True
+                    'monitoring': {
+                        'create_sns_topic': True
                     }
                 }
             },
             'lambda': {
                 'athena_partition_refresh_config': {
                     'enabled': True,
+                    'enable_metrics': True,
                     'current_version': '$LATEST',
                     'refresh_type': {
                         'repair_hive_table': {