[pr] addressing feedback from jacknagz and austinbyers, take 2

conf/lambda.json

@@ -24,18 +24,16 @@
     }
   },
   "athena_partition_refresh_config": {
+    "buckets": {
+      "PREFIX_GOES_HERE.streamalerts": "alerts"
+    },
     "current_version": "$LATEST",
     "enable_metrics": false,
-    "enabled": true,
     "handler": "stream_alert.athena_partition_refresh.main.handler",
     "log_level": "info",
-    "buckets": {
-      "PREFIX_GOES_HERE.streamalerts": "alerts"
-    },
     "source_bucket": "PREFIX_GOES_HERE.streamalert.source",
     "source_current_hash": "<auto_generated>",
-    "source_object_key": "<auto_generated>",
-    "third_party_libraries": []
+    "source_object_key": "<auto_generated>"
   },
   "rule_processor_config": {
     "handler": "stream_alert.rule_processor.main.handler",

docs/source/athena-arch.rst

@@ -62,7 +62,6 @@ Key                                  Required  Default                Descriptio
 
   {
     "athena_partition_refresh_config": {
-      "enabled": true,
       "enable_metrics": false,
       "log_level": "info",
       "memory": 128,
@@ -89,7 +88,7 @@ Going forward, if the deploy flag ``--processor all`` is used, it will redeploy
 Monitoring
 ~~~~~~~~~~
 
-To ensure the function is operating as expected, monitor the following SQS metrics for ``<prefix>_streamalert_athena_data_bucket_notifications``:
+To ensure the function is operating as expected, monitor the following SQS metrics for ``<prefix>_streamalert_athena_s3_notifications``:
 
 * ``NumberOfMessagesReceived``
 * ``NumberOfMessagesSent``

manage.py

@@ -1268,10 +1268,13 @@ def _add_athena_create_table_subparser(athena_subparsers):
     # Validate the provided schema-override options
     def _validate_override(val):
         """Make sure the input is in the format column_name=type"""
+        err = ('Invalid override expression [{}]. The proper format is '
+               '"column_name=value_type"').format(val)
         if not '=' in val:
-            raise athena_create_table_parser.error(
-                'Invalid override expression [{}]. The proper format is '
-                '"column_name=value_type"'.format(val))
+            raise athena_create_table_parser.error(err)
+
+        if len(val.split('=')) != 2:
+            raise athena_create_table_parser.error(err)
 
     athena_create_table_parser.add_argument(
         '--schema-override',

stream_alert/athena_partition_refresh/main.py

@@ -264,7 +264,7 @@ def check_database_exists(self, **kwargs):
 
         return False
 
-    def check_table_exists(self, table_name):
+    def check_table_exists(self, table_name, creating_table=False):
         """Verify a given StreamAlert Athena table exists."""
         query_success, query_resp = self.run_athena_query(
             query='SHOW TABLES LIKE \'{}\';'.format(table_name),
@@ -274,9 +274,10 @@ def check_table_exists(self, table_name):
         if query_success and query_resp['ResultSet']['Rows']:
             return True
 
-        LOGGER.info('The streamalert table \'%s\' does not exist.', table_name)
-        LOGGER.info('For help with creating tables: '
-                    '$ python manage.py athena create-table --help')
+        if not creating_table:
+            LOGGER.info('The streamalert table \'%s\' does not exist.', table_name)
+            LOGGER.info('For help with creating tables: '
+                        '$ python manage.py athena create-table --help')
         return False
 
     def add_partition(self, s3_buckets_and_keys):

stream_alert_cli/athena/handler.py

@@ -221,7 +221,7 @@ def create_table(table, bucket, table_type, config, schema_override=None):
             return
 
         # Check if the table exists
-        if athena_client.check_table_exists(sanitized_table_name):
+        if athena_client.check_table_exists(sanitized_table_name, True):
             LOGGER_CLI.info('The \'%s\' table already exists.', sanitized_table_name)
             return
 
@@ -265,7 +265,7 @@ def create_table(table, bucket, table_type, config, schema_override=None):
             schema=athena_schema, table_name=sanitized_table_name, bucket=bucket)
 
     elif table_type == 'alerts':
-        if athena_client.check_table_exists(table_type):
+        if athena_client.check_table_exists(table_type, True):
             LOGGER_CLI.info('The \'alerts\' table already exists.')
             return
         query = ALERTS_TABLE_STATEMENT.format(bucket=bucket)

stream_alert_cli/config.py

@@ -69,7 +69,6 @@ def generate_athena(self):
         prefix = self.config['global']['account']['prefix']
 
         athena_config_template = {
-            'enabled': True,
             'enable_metrics': False,
             'current_version': '$LATEST',
             'buckets': {

stream_alert_cli/terraform/athena.py

@@ -34,13 +34,15 @@ def generate_athena(config):
     prefix = config['global']['account']['prefix']
     database = athena_config.get('database_name', '{}_streamalert'.format(prefix))
 
-    results_bucket_name = athena_config.get('results_bucket', '').strip()
-    if results_bucket_name == '':
-        results_bucket_name = '{}.streamalert.athena-results'.format(prefix)
-
-    queue_name = athena_config.get('queue_name', '').strip()
-    if queue_name == '':
-        queue_name = '{}_streamalert_athena_data_bucket_notifications'.format(prefix)
+    results_bucket_name = athena_config.get(
+        'results_bucket',
+        '{}.streamalert.athena-results'.format(prefix)
+    ).strip()
+
+    queue_name = athena_config.get(
+        'queue_name',
+        '{}_streamalert_athena_s3_notifications'.format(prefix)
+    ).strip()
 
     athena_dict['module']['stream_alert_athena'] = {
         's3_logging_bucket': '{}.streamalert.s3-logging'.format(prefix),

stream_alert_cli/terraform/handler.py

@@ -124,9 +124,8 @@ def _terraform_init(config):
     deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters'])
 
     LOGGER_CLI.info('Deploying Lambda Functions')
-    # deploy both lambda functions
+
     deploy(deploy_opts(['rule', 'alert', 'athena'], []), config)
-    # create all remainder infrastructure
 
     LOGGER_CLI.info('Building Remainder Infrastructure')
     tf_runner(refresh=False)

tests/unit/conf/lambda.json

@@ -33,8 +33,7 @@
       "unit-testing-2.streamalerts": "alerts"
     },
     "current_version": "$LATEST",
-    "enable_metrics": true,
-    "enabled": true,
+    "enable_metrics": false,
     "handler": "main.handler",
     "memory": "128",
     "source_bucket": "unit-testing.streamalert.source",

tests/unit/helpers/base.py

@@ -128,7 +128,6 @@ def basic_streamalert_config():
             'athena_partition_refresh_config': {
                 'current_version': '$LATEST',
                 'enable_metrics': False,
-                'enabled': True,
                 'handler': 'main.handler',
                 'memory': 128,
                 'partitioning': {

tests/unit/stream_alert_athena_partition_refresh/test_main.py

@@ -89,7 +89,6 @@
             ]
         },
         'athena_partition_refresh_config': {
-            'enabled': True,
             'buckets': {
                 'unit-testing.streamalerts': 'alerts',
                 'unit-testing.streamalert.data': 'data'

tests/unit/stream_alert_cli/terraform/test_athena.py

@@ -24,7 +24,6 @@ def test_generate_athena():
     """CLI - Terraform Generate Athena"""
 
     CONFIG['lambda']['athena_partition_refresh_config'] = {
-        'enabled': True,
         'current_version': '$LATEST',
         'buckets': {
             'unit-testing.streamalerts': 'alerts',
@@ -47,8 +46,7 @@ def test_generate_athena():
                 's3_logging_bucket': '{}.streamalert.s3-logging'.format(prefix),
                 'source': 'modules/tf_stream_alert_athena',
                 'database_name': '{}_streamalert'.format(prefix),
-                'queue_name':
-                '{}_streamalert_athena_data_bucket_notifications'.format(prefix),
+                'queue_name': '{}_streamalert_athena_s3_notifications'.format(prefix),
                 'results_bucket': '{}.streamalert.athena-results'.format(prefix),
                 'current_version': '$LATEST',
                 'enable_metrics': False,