From 43914c50894fa047de5ca5889caaed43bb428731 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 14:09:43 -0800 Subject: [PATCH 1/8] Support SNS and SQS when mocking alert outputs --- stream_alert_cli/helpers.py | 12 ++++++++---- stream_alert_cli/test.py | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/stream_alert_cli/helpers.py b/stream_alert_cli/helpers.py index d5b0e2f53..1a13a9148 100644 --- a/stream_alert_cli/helpers.py +++ b/stream_alert_cli/helpers.py @@ -29,11 +29,13 @@ from botocore.exceptions import ClientError from moto import ( mock_cloudwatch, - mock_kms, + mock_dynamodb2, mock_kinesis, + mock_kms, mock_lambda, mock_s3, - mock_dynamodb2, + mock_sns, + mock_sqs ) from stream_alert_cli.logger import LOGGER_CLI @@ -522,10 +524,12 @@ def wrap(func): """Wrap the returned function with or without mocks""" if context.mocked: @mock_cloudwatch + @mock_kinesis + @mock_kms @mock_lambda @mock_s3 - @mock_kms - @mock_kinesis + @mock_sns + @mock_sqs def mocked(options, context): """This function is now mocked using moto mock decorators to override any boto3 calls. Wrapping this function here allows diff --git a/stream_alert_cli/test.py b/stream_alert_cli/test.py index f84f6ad66..61c1af430 100644 --- a/stream_alert_cli/test.py +++ b/stream_alert_cli/test.py @@ -772,6 +772,15 @@ def setup_outputs(self, alert): lambda_function = parts[-1] helpers.create_lambda_function(lambda_function, self.region) + + elif service == 'aws-sns': + topic_name = self.outputs_config[service][descriptor] + boto3.client('sns', region_name=self.region).create_topic(Name=topic_name) + + elif service == 'aws-sqs': + queue_name = self.outputs_config[service][descriptor] + boto3.client('sqs', region_name=self.region).create_queue(QueueName=queue_name) + elif service == 'komand': output_name = '{}/{}'.format(service, descriptor) creds = {'komand_auth_token': '00000000-0000-0000-0000-000000000000', From 055e3ca5666dee23df952ad103c9eec2947673b8 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 14:13:12 -0800 Subject: [PATCH 2/8] Use service-role for threat intel autoscaling --- .../tf_threat_intel_downloader/dynamodb.tf | 79 +------------------ 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/terraform/modules/tf_threat_intel_downloader/dynamodb.tf b/terraform/modules/tf_threat_intel_downloader/dynamodb.tf index ee4f2846d..679663efd 100644 --- a/terraform/modules/tf_threat_intel_downloader/dynamodb.tf +++ b/terraform/modules/tf_threat_intel_downloader/dynamodb.tf @@ -26,89 +26,12 @@ resource "aws_dynamodb_table" "threat_intel_ioc" { } } -// IAM Role: Application autoscalling role -resource "aws_iam_role" "stream_alert_dynamodb_appautoscaling" { - count = "${var.autoscale ? 1 : 0}" - name = "${var.prefix}_streamalert_dynamodb_appautoscaling" - assume_role_policy = "${data.aws_iam_policy_document.appautoscaling_assume_role_policy.json}" -} - -// IAM Policy Doc: Generic Application Autoscaling AssumeRole -data "aws_iam_policy_document" "appautoscaling_assume_role_policy" { - count = "${var.autoscale ? 1 : 0}" - - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["application-autoscaling.amazonaws.com"] - } - } -} - -// IAM Role Policy: Allow appautoscaling IAM role to autoscaling DynamoDB table -resource "aws_iam_role_policy" "appautoscaling_update_table" { - count = "${var.autoscale ? 1 : 0}" - name = "DynamoDBAppAutoscaleUpdateTablePolicy" - role = "${aws_iam_role.stream_alert_dynamodb_appautoscaling.id}" - policy = "${data.aws_iam_policy_document.appautoscaling_update_table.json}" -} - -// IAM Policy Doc: Allow autoscaling IAM role to send alarm to CloudWatch -// and change table settings for autoscaling. -// This policy is allow the role to change table settings -data "aws_iam_policy_document" "appautoscaling_update_table" { - count = "${var.autoscale ? 1 : 0}" - - statement { - effect = "Allow" - - actions = [ - "dynamodb:DescribeTable", - "dynamodb:UpdateTable", - ] - - resources = [ - "${aws_dynamodb_table.threat_intel_ioc.arn}", - ] - } -} - -// IAM Role Policy: Allow appautoscaling IAM role to autoscaling DynamoDB table -resource "aws_iam_role_policy" "appautoscaling_cloudwatch_alarms" { - count = "${var.autoscale ? 1 : 0}" - name = "DynamoDBAppAutoscaleCloudWatchAlarmsPolicy" - role = "${aws_iam_role.stream_alert_dynamodb_appautoscaling.id}" - policy = "${data.aws_iam_policy_document.appautoscaling_cloudwatch_alarms.json}" -} - -// IAM Policy Doc: This policy is allow the role to send alarm to CloudWatch. -data "aws_iam_policy_document" "appautoscaling_cloudwatch_alarms" { - count = "${var.autoscale ? 1 : 0}" - - statement { - effect = "Allow" - - actions = [ - "cloudwatch:PutMetricAlarm", - "cloudwatch:DescribeAlarms", - "cloudwatch:GetMetricStatistics", - "cloudwatch:SetAlarmState", - "cloudwatch:DeleteAlarms", - ] - - resources = ["*"] - } -} - resource "aws_appautoscaling_target" "dynamodb_table_read_target" { count = "${var.autoscale ? 1 : 0}" max_capacity = "${var.max_read_capacity}" min_capacity = "${var.min_read_capacity}" resource_id = "table/${var.prefix}_streamalert_threat_intel_downloader" - role_arn = "${aws_iam_role.stream_alert_dynamodb_appautoscaling.arn}" + role_arn = "arn:aws:iam::${var.account_id}:role/aws-service-role/dynamodb.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" scalable_dimension = "dynamodb:table:ReadCapacityUnits" service_namespace = "dynamodb" } From ba317f53b2d061d8913d23c58a87d13e6087cc85 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 14:13:32 -0800 Subject: [PATCH 3/8] Terraform format --- terraform/modules/tf_stream_alert_globals/main.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/terraform/modules/tf_stream_alert_globals/main.tf b/terraform/modules/tf_stream_alert_globals/main.tf index 0560486da..6c884907f 100644 --- a/terraform/modules/tf_stream_alert_globals/main.tf +++ b/terraform/modules/tf_stream_alert_globals/main.tf @@ -19,19 +19,16 @@ resource "aws_dynamodb_table" "alerts_table" { name = "RuleName" type = "S" } - attribute { name = "Timestamp" type = "S" } - // Enable expriation time while testing Dynamo table for alerts // TODO: Remove TTL once Alert Merger is implemented ttl { attribute_name = "TTL" enabled = true } - tags { Name = "StreamAlert" } From 9b3e4e4a294c498f5789350ed81ca5ce8deee5e6 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 14:32:15 -0800 Subject: [PATCH 4/8] Remove third_party_libraries from all but rule processor --- conf/lambda.json | 4 +--- stream_alert_cli/manage_lambda/package.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/lambda.json b/conf/lambda.json index cfa637791..01c996ac8 100644 --- a/conf/lambda.json +++ b/conf/lambda.json @@ -17,7 +17,6 @@ "source_bucket": "PREFIX_GOES_HERE.streamalert.source", "source_current_hash": "", "source_object_key": "", - "third_party_libraries": [], "timeout": 60, "vpc_config": { "security_group_ids": [], @@ -37,7 +36,6 @@ "handler": "app_integrations.main.handler", "source_bucket": "PREFIX_GOES_HERE.streamalert.source", "source_current_hash": "", - "source_object_key": "", - "third_party_libraries": [] + "source_object_key": "" } } \ No newline at end of file diff --git a/stream_alert_cli/manage_lambda/package.py b/stream_alert_cli/manage_lambda/package.py index 932eeb211..89d7e9eb5 100644 --- a/stream_alert_cli/manage_lambda/package.py +++ b/stream_alert_cli/manage_lambda/package.py @@ -231,7 +231,7 @@ def _resolve_third_party(self, temp_package_path): # Add any custom libs needed by rules, etc third_party_libs.update( - set(self.config['lambda'][self.config_key]['third_party_libraries'])) + set(self.config['lambda'][self.config_key].get('third_party_libraries', []))) # Return a default of True here if no libraries to install if not third_party_libs: From d089b8b4fdee12be4336fdade56c61eb9ec95004 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 15:01:41 -0800 Subject: [PATCH 5/8] Add BinaryAlert YARA match rule --- conf/sources.json | 7 +++ rules/community/binaryalert/__init__.py | 0 .../binaryalert/binaryalert_yara_match.py | 15 ++++++ .../binaryalert/binaryalert_yara_match.json | 50 +++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 rules/community/binaryalert/__init__.py create mode 100644 rules/community/binaryalert/binaryalert_yara_match.py create mode 100644 tests/integration/rules/binaryalert/binaryalert_yara_match.json diff --git a/conf/sources.json b/conf/sources.json index 0b0f2f8a9..60beb07ac 100644 --- a/conf/sources.json +++ b/conf/sources.json @@ -16,6 +16,13 @@ ] } }, + "sns": { + "prefix_cluster_sample_topic": { + "logs": [ + "binaryalert" + ] + } + }, "stream_alert_app": { "prefix_cluster_box_admin_events_sm-app-name_app": { "logs": [ diff --git a/rules/community/binaryalert/__init__.py b/rules/community/binaryalert/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rules/community/binaryalert/binaryalert_yara_match.py b/rules/community/binaryalert/binaryalert_yara_match.py new file mode 100644 index 000000000..7b4a489ed --- /dev/null +++ b/rules/community/binaryalert/binaryalert_yara_match.py @@ -0,0 +1,15 @@ +"""Alert on destructive AWS API calls.""" +from stream_alert.rule_processor.rules_engine import StreamRules + +rule = StreamRules.rule + + +@rule(logs=['binaryalert'], + outputs=['aws-sns:sample-topic']) +def binaryalert_yara_match(rec): + """ + author: Austin Byers (Airbnb CSIRT) + description: BinaryAlert found a binary matching a YARA rule + reference: https://binaryalert.io + """ + return rec['NumMatchedRules'] > 0 diff --git a/tests/integration/rules/binaryalert/binaryalert_yara_match.json b/tests/integration/rules/binaryalert/binaryalert_yara_match.json new file mode 100644 index 000000000..21c84b80e --- /dev/null +++ b/tests/integration/rules/binaryalert/binaryalert_yara_match.json @@ -0,0 +1,50 @@ +{ + "records": [ + { + "data": { + "FileInfo": { + "MD5": "...", + "S3LastModified": "...", + "S3Location": "...", + "S3Metadata": {}, + "SHA256": "..." + }, + "MatchedRules": { + "Rule1": { + "MatchedStrings": [ + "$eicar_regex" + ], + "Meta": { + "author": "Austin Byers (Airbnb CSIRT)", + "description": "This is a standard AV test, intended to check whether BinaryAlert is working correctly.", + "reference": "http://www.eicar.org/86-0-Intended-use.html" + }, + "RuleFile": "eicar.yar", + "RuleName": "eicar_av_test", + "RuleTags": [] + } + }, + "NumMatchedRules": "1" + }, + "description": "All YARA matches from BinaryAlert trigger an alert", + "log": "binaryalert", + "service": "sns", + "source": "prefix_cluster_sample_topic", + "trigger_rules": [ + "binaryalert_yara_match" + ] + }, + { + "data": { + "FileInfo": {}, + "MatchedRules": {}, + "NumMatchedRules": "0" + }, + "description": "No alerts triggered if no YARA rules were matched", + "log": "binaryalert", + "service": "sns", + "source": "prefix_cluster_sample_topic", + "trigger_rules": [] + } + ] +} From 940ba64791bbfcf2eeec3bd59a10851824c0c08b Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 15:17:41 -0800 Subject: [PATCH 6/8] Speed up slowest unit test --- .../stream_alert_athena_partition_refresh/test_main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/stream_alert_athena_partition_refresh/test_main.py b/tests/unit/stream_alert_athena_partition_refresh/test_main.py index 26b5ebee7..4599d6788 100644 --- a/tests/unit/stream_alert_athena_partition_refresh/test_main.py +++ b/tests/unit/stream_alert_athena_partition_refresh/test_main.py @@ -175,14 +175,13 @@ def test_handler_no_received_messages( @patch('stream_alert.athena_partition_refresh.main.LOGGER') @patch('stream_alert.athena_partition_refresh.main._load_config', return_value=CONFIG_DATA) - @patch('stream_alert.athena_partition_refresh.main.' - 'StreamAlertSQSClient.unique_s3_buckets_and_keys', - return_value={}) + @patch('stream_alert.athena_partition_refresh.main.StreamAlertSQSClient') @mock_sqs - def test_handler_no_unique_buckets(self, _, mock_config, mock_logging): + def test_handler_no_unique_buckets(self, mock_sqs_client, mock_config, mock_logging): """Athena - Handler - No Unique Buckets""" test_sqs_client = TestStreamAlertSQSClient() test_sqs_client.setup() + mock_sqs_client.return_value.unique_s3_buckets_and_keys = lambda: {} handler(None, None) From 02533c42136326117578d4857a2c6a8659306359 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 15:42:48 -0800 Subject: [PATCH 7/8] Fix RuntimeWarning import error --- stream_alert/alert_processor/main.py | 1 + stream_alert/athena_partition_refresh/main.py | 1 + stream_alert/rule_processor/main.py | 1 + stream_alert/threat_intel_downloader/main.py | 1 + 4 files changed, 4 insertions(+) diff --git a/stream_alert/alert_processor/main.py b/stream_alert/alert_processor/main.py index d29cee206..a3eb39077 100644 --- a/stream_alert/alert_processor/main.py +++ b/stream_alert/alert_processor/main.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +from __future__ import absolute_import # Suppresses RuntimeWarning import error in Lambda from collections import OrderedDict import json diff --git a/stream_alert/athena_partition_refresh/main.py b/stream_alert/athena_partition_refresh/main.py index cb5f1711c..05cbade36 100644 --- a/stream_alert/athena_partition_refresh/main.py +++ b/stream_alert/athena_partition_refresh/main.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +from __future__ import absolute_import # Suppresses RuntimeWarning import error in Lambda from collections import defaultdict from datetime import datetime import json diff --git a/stream_alert/rule_processor/main.py b/stream_alert/rule_processor/main.py index 6d96d02ee..c8a2376b5 100644 --- a/stream_alert/rule_processor/main.py +++ b/stream_alert/rule_processor/main.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +from __future__ import absolute_import # Suppresses RuntimeWarning import error in Lambda import importlib import os diff --git a/stream_alert/threat_intel_downloader/main.py b/stream_alert/threat_intel_downloader/main.py index 230c7af71..e35ba47a0 100644 --- a/stream_alert/threat_intel_downloader/main.py +++ b/stream_alert/threat_intel_downloader/main.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +from __future__ import absolute_import # Suppresses RuntimeWarning import error in Lambda import json import os From b872973209dd70b6af397e41e653bf18c5637589 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Tue, 6 Mar 2018 17:52:41 -0800 Subject: [PATCH 8/8] Remove now-redundant autoscale variable --- stream_alert_cli/terraform/threat_intel_downloader.py | 1 - terraform/modules/tf_threat_intel_downloader/dynamodb.tf | 4 +--- terraform/modules/tf_threat_intel_downloader/variables.tf | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/stream_alert_cli/terraform/threat_intel_downloader.py b/stream_alert_cli/terraform/threat_intel_downloader.py index c4cd5115a..ba0964a9e 100644 --- a/stream_alert_cli/terraform/threat_intel_downloader.py +++ b/stream_alert_cli/terraform/threat_intel_downloader.py @@ -55,7 +55,6 @@ def generate_threat_intel_downloader(config): ['expiration_ts', 'itype', 'source', 'type', 'value']), 'ioc_filters': ti_downloader_config.get('ioc_filters', ['crowdstrike', '@airbnb.com']), 'ioc_types': ti_downloader_config.get('ioc_types', ['domain', 'ip', 'md5']), - 'autoscale': ti_downloader_config.get('autoscale', False), 'max_read_capacity': ti_downloader_config.get('max_read_capacity', '5'), 'min_read_capacity': ti_downloader_config.get('min_read_capacity', '5'), 'target_utilization': ti_downloader_config.get('target_utilization', '70') diff --git a/terraform/modules/tf_threat_intel_downloader/dynamodb.tf b/terraform/modules/tf_threat_intel_downloader/dynamodb.tf index 679663efd..27f947f6e 100644 --- a/terraform/modules/tf_threat_intel_downloader/dynamodb.tf +++ b/terraform/modules/tf_threat_intel_downloader/dynamodb.tf @@ -27,17 +27,15 @@ resource "aws_dynamodb_table" "threat_intel_ioc" { } resource "aws_appautoscaling_target" "dynamodb_table_read_target" { - count = "${var.autoscale ? 1 : 0}" max_capacity = "${var.max_read_capacity}" min_capacity = "${var.min_read_capacity}" - resource_id = "table/${var.prefix}_streamalert_threat_intel_downloader" + resource_id = "table/${aws_dynamodb_table.threat_intel_ioc.name}" role_arn = "arn:aws:iam::${var.account_id}:role/aws-service-role/dynamodb.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_DynamoDBTable" scalable_dimension = "dynamodb:table:ReadCapacityUnits" service_namespace = "dynamodb" } resource "aws_appautoscaling_policy" "dynamodb_table_read_policy" { - count = "${var.autoscale ? 1 : 0}" name = "DynamoDBReadCapacityUtilization:${aws_appautoscaling_target.dynamodb_table_read_target.resource_id}" policy_type = "TargetTrackingScaling" resource_id = "${aws_appautoscaling_target.dynamodb_table_read_target.resource_id}" diff --git a/terraform/modules/tf_threat_intel_downloader/variables.tf b/terraform/modules/tf_threat_intel_downloader/variables.tf index f0d16911c..62b11a07c 100644 --- a/terraform/modules/tf_threat_intel_downloader/variables.tf +++ b/terraform/modules/tf_threat_intel_downloader/variables.tf @@ -71,10 +71,6 @@ variable "log_retention" { default = 14 } -variable "autoscale" { - default = false -} - variable "max_read_capacity" { default = 5 }