From bfff54a3214dd919989ee9170a0e6aa90fa184c4 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Tue, 13 Mar 2018 11:29:46 -0700 Subject: [PATCH] [pr] addressing feedback from jacknagz and austinbyers, take 2 --- conf/lambda.json | 10 ++++------ docs/source/athena-arch.rst | 3 +-- manage.py | 9 ++++++--- stream_alert/athena_partition_refresh/main.py | 9 +++++---- stream_alert_cli/athena/handler.py | 4 ++-- stream_alert_cli/config.py | 1 - stream_alert_cli/terraform/athena.py | 16 +++++++++------- stream_alert_cli/terraform/handler.py | 3 +-- tests/unit/conf/lambda.json | 3 +-- tests/unit/helpers/base.py | 1 - .../test_main.py | 1 - .../stream_alert_cli/terraform/test_athena.py | 4 +--- 12 files changed, 30 insertions(+), 34 deletions(-) diff --git a/conf/lambda.json b/conf/lambda.json index 0b4533732..a10ba4c48 100644 --- a/conf/lambda.json +++ b/conf/lambda.json @@ -24,18 +24,16 @@ } }, "athena_partition_refresh_config": { + "buckets": { + "PREFIX_GOES_HERE.streamalerts": "alerts" + }, "current_version": "$LATEST", "enable_metrics": false, - "enabled": true, "handler": "stream_alert.athena_partition_refresh.main.handler", "log_level": "info", - "buckets": { - "PREFIX_GOES_HERE.streamalerts": "alerts" - }, "source_bucket": "PREFIX_GOES_HERE.streamalert.source", "source_current_hash": "", - "source_object_key": "", - "third_party_libraries": [] + "source_object_key": "" }, "rule_processor_config": { "handler": "stream_alert.rule_processor.main.handler", diff --git a/docs/source/athena-arch.rst b/docs/source/athena-arch.rst index f121b1b3b..5b3b13e67 100644 --- a/docs/source/athena-arch.rst +++ b/docs/source/athena-arch.rst @@ -62,7 +62,6 @@ Key Required Default Descriptio { "athena_partition_refresh_config": { - "enabled": true, "enable_metrics": false, "log_level": "info", "memory": 128, @@ -89,7 +88,7 @@ Going forward, if the deploy flag ``--processor all`` is used, it will redeploy Monitoring ~~~~~~~~~~ -To ensure the function is operating as expected, monitor the following SQS metrics for ``_streamalert_athena_data_bucket_notifications``: +To ensure the function is operating as expected, monitor the following SQS metrics for ``_streamalert_athena_s3_notifications``: * ``NumberOfMessagesReceived`` * ``NumberOfMessagesSent`` diff --git a/manage.py b/manage.py index 7c3e2b171..4b43b92cd 100755 --- a/manage.py +++ b/manage.py @@ -1268,10 +1268,13 @@ def _add_athena_create_table_subparser(athena_subparsers): # Validate the provided schema-override options def _validate_override(val): """Make sure the input is in the format column_name=type""" + err = ('Invalid override expression [{}]. The proper format is ' + '"column_name=value_type"').format(val) if not '=' in val: - raise athena_create_table_parser.error( - 'Invalid override expression [{}]. The proper format is ' - '"column_name=value_type"'.format(val)) + raise athena_create_table_parser.error(err) + + if len(val.split('=')) != 2: + raise athena_create_table_parser.error(err) athena_create_table_parser.add_argument( '--schema-override', diff --git a/stream_alert/athena_partition_refresh/main.py b/stream_alert/athena_partition_refresh/main.py index ba14a84a6..50aae8d98 100644 --- a/stream_alert/athena_partition_refresh/main.py +++ b/stream_alert/athena_partition_refresh/main.py @@ -264,7 +264,7 @@ def check_database_exists(self, **kwargs): return False - def check_table_exists(self, table_name): + def check_table_exists(self, table_name, creating_table=False): """Verify a given StreamAlert Athena table exists.""" query_success, query_resp = self.run_athena_query( query='SHOW TABLES LIKE \'{}\';'.format(table_name), @@ -274,9 +274,10 @@ def check_table_exists(self, table_name): if query_success and query_resp['ResultSet']['Rows']: return True - LOGGER.info('The streamalert table \'%s\' does not exist.', table_name) - LOGGER.info('For help with creating tables: ' - '$ python manage.py athena create-table --help') + if not creating_table: + LOGGER.info('The streamalert table \'%s\' does not exist.', table_name) + LOGGER.info('For help with creating tables: ' + '$ python manage.py athena create-table --help') return False def add_partition(self, s3_buckets_and_keys): diff --git a/stream_alert_cli/athena/handler.py b/stream_alert_cli/athena/handler.py index 05243ea86..e037dfb5a 100644 --- a/stream_alert_cli/athena/handler.py +++ b/stream_alert_cli/athena/handler.py @@ -221,7 +221,7 @@ def create_table(table, bucket, table_type, config, schema_override=None): return # Check if the table exists - if athena_client.check_table_exists(sanitized_table_name): + if athena_client.check_table_exists(sanitized_table_name, True): LOGGER_CLI.info('The \'%s\' table already exists.', sanitized_table_name) return @@ -265,7 +265,7 @@ def create_table(table, bucket, table_type, config, schema_override=None): schema=athena_schema, table_name=sanitized_table_name, bucket=bucket) elif table_type == 'alerts': - if athena_client.check_table_exists(table_type): + if athena_client.check_table_exists(table_type, True): LOGGER_CLI.info('The \'alerts\' table already exists.') return query = ALERTS_TABLE_STATEMENT.format(bucket=bucket) diff --git a/stream_alert_cli/config.py b/stream_alert_cli/config.py index c0767e687..a3e4aafdb 100644 --- a/stream_alert_cli/config.py +++ b/stream_alert_cli/config.py @@ -69,7 +69,6 @@ def generate_athena(self): prefix = self.config['global']['account']['prefix'] athena_config_template = { - 'enabled': True, 'enable_metrics': False, 'current_version': '$LATEST', 'buckets': { diff --git a/stream_alert_cli/terraform/athena.py b/stream_alert_cli/terraform/athena.py index 2060f8aee..2fdc52279 100644 --- a/stream_alert_cli/terraform/athena.py +++ b/stream_alert_cli/terraform/athena.py @@ -34,13 +34,15 @@ def generate_athena(config): prefix = config['global']['account']['prefix'] database = athena_config.get('database_name', '{}_streamalert'.format(prefix)) - results_bucket_name = athena_config.get('results_bucket', '').strip() - if results_bucket_name == '': - results_bucket_name = '{}.streamalert.athena-results'.format(prefix) - - queue_name = athena_config.get('queue_name', '').strip() - if queue_name == '': - queue_name = '{}_streamalert_athena_data_bucket_notifications'.format(prefix) + results_bucket_name = athena_config.get( + 'results_bucket', + '{}.streamalert.athena-results'.format(prefix) + ).strip() + + queue_name = athena_config.get( + 'queue_name', + '{}_streamalert_athena_s3_notifications'.format(prefix) + ).strip() athena_dict['module']['stream_alert_athena'] = { 's3_logging_bucket': '{}.streamalert.s3-logging'.format(prefix), diff --git a/stream_alert_cli/terraform/handler.py b/stream_alert_cli/terraform/handler.py index 70a635334..f49bacae8 100644 --- a/stream_alert_cli/terraform/handler.py +++ b/stream_alert_cli/terraform/handler.py @@ -124,9 +124,8 @@ def _terraform_init(config): deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters']) LOGGER_CLI.info('Deploying Lambda Functions') - # deploy both lambda functions + deploy(deploy_opts(['rule', 'alert', 'athena'], []), config) - # create all remainder infrastructure LOGGER_CLI.info('Building Remainder Infrastructure') tf_runner(refresh=False) diff --git a/tests/unit/conf/lambda.json b/tests/unit/conf/lambda.json index 5cab9f643..9bde395b9 100644 --- a/tests/unit/conf/lambda.json +++ b/tests/unit/conf/lambda.json @@ -33,8 +33,7 @@ "unit-testing-2.streamalerts": "alerts" }, "current_version": "$LATEST", - "enable_metrics": true, - "enabled": true, + "enable_metrics": false, "handler": "main.handler", "memory": "128", "source_bucket": "unit-testing.streamalert.source", diff --git a/tests/unit/helpers/base.py b/tests/unit/helpers/base.py index d92bea8bd..8df70b8f1 100644 --- a/tests/unit/helpers/base.py +++ b/tests/unit/helpers/base.py @@ -128,7 +128,6 @@ def basic_streamalert_config(): 'athena_partition_refresh_config': { 'current_version': '$LATEST', 'enable_metrics': False, - 'enabled': True, 'handler': 'main.handler', 'memory': 128, 'partitioning': { diff --git a/tests/unit/stream_alert_athena_partition_refresh/test_main.py b/tests/unit/stream_alert_athena_partition_refresh/test_main.py index 00b173d9f..0979c9d53 100644 --- a/tests/unit/stream_alert_athena_partition_refresh/test_main.py +++ b/tests/unit/stream_alert_athena_partition_refresh/test_main.py @@ -89,7 +89,6 @@ ] }, 'athena_partition_refresh_config': { - 'enabled': True, 'buckets': { 'unit-testing.streamalerts': 'alerts', 'unit-testing.streamalert.data': 'data' diff --git a/tests/unit/stream_alert_cli/terraform/test_athena.py b/tests/unit/stream_alert_cli/terraform/test_athena.py index 28d9e46b4..992864a88 100644 --- a/tests/unit/stream_alert_cli/terraform/test_athena.py +++ b/tests/unit/stream_alert_cli/terraform/test_athena.py @@ -24,7 +24,6 @@ def test_generate_athena(): """CLI - Terraform Generate Athena""" CONFIG['lambda']['athena_partition_refresh_config'] = { - 'enabled': True, 'current_version': '$LATEST', 'buckets': { 'unit-testing.streamalerts': 'alerts', @@ -47,8 +46,7 @@ def test_generate_athena(): 's3_logging_bucket': '{}.streamalert.s3-logging'.format(prefix), 'source': 'modules/tf_stream_alert_athena', 'database_name': '{}_streamalert'.format(prefix), - 'queue_name': - '{}_streamalert_athena_data_bucket_notifications'.format(prefix), + 'queue_name': '{}_streamalert_athena_s3_notifications'.format(prefix), 'results_bucket': '{}.streamalert.athena-results'.format(prefix), 'current_version': '$LATEST', 'enable_metrics': False,