Merge pull request #643 from airbnb/ryandeivert-default-output

[outputs] adding support for required alerting outputs
airbnb · Mar 20, 2018 · e6bd483 · e6bd483
2 parents a97fb60 + da8192a
commit e6bd483
Show file tree

Hide file tree

Showing 35 changed files with 202 additions and 62 deletions.
diff --git a/rules/community/binaryalert/binaryalert_yara_match.py b/rules/community/binaryalert/binaryalert_yara_match.py
@@ -4,9 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(
-    logs=['binaryalert'],
-    outputs=['aws-firehose:alerts'])
+@rule(logs=['binaryalert'])
 def binaryalert_yara_match(rec):
     """
     author:       Austin Byers (Airbnb CSIRT)

diff --git a/rules/community/cloudtrail/cloudtrail_critical_api_calls.py b/rules/community/cloudtrail/cloudtrail_critical_api_calls.py
@@ -6,9 +6,7 @@
 disable = StreamRules.disable()
 
 
-@rule(
-    logs=['cloudtrail:events'],
-    outputs=['aws-firehose:alerts'])
+@rule(logs=['cloudtrail:events'])
 def cloudtrail_critical_api_calls(rec):
     """
     author:           airbnb_csirt

diff --git a/rules/community/cloudtrail/cloudtrail_mfa_policy_abuse_attempt.py b/rules/community/cloudtrail/cloudtrail_mfa_policy_abuse_attempt.py
@@ -6,9 +6,7 @@
 disable = StreamRules.disable()
 
 
-@rule(
-    logs=['cloudtrail:events'],
-    outputs=['aws-firehose:alerts'])
+@rule(logs=['cloudtrail:events'])
 def cloudtrail_mfa_policy_abuse_attempt(rec):
     """
     author:           Scott Piper of Summit Route in collaboration with Duo Security

diff --git a/rules/community/cloudtrail/cloudtrail_network_acl_ingress_anywhere.py b/rules/community/cloudtrail/cloudtrail_network_acl_ingress_anywhere.py
@@ -6,7 +6,6 @@
 
 @rule(
     logs=['cloudwatch:events'],
-    outputs=['aws-firehose:alerts'],
     req_subkeys={
         'detail': ['eventName', 'requestParameters']
     })

diff --git a/rules/community/cloudtrail/cloudtrail_put_bucket_acl.py b/rules/community/cloudtrail/cloudtrail_put_bucket_acl.py
@@ -6,7 +6,6 @@
 
 @rule(
     logs=['cloudwatch:events'],
-    outputs=['aws-firehose:alerts'],
     req_subkeys={
         'detail': ['requestParameters', 'eventName']
     })

diff --git a/rules/community/cloudtrail/cloudtrail_put_object_acl_public.py b/rules/community/cloudtrail/cloudtrail_put_object_acl_public.py
@@ -7,7 +7,6 @@
 
 @rule(
     logs=['cloudwatch:events'],
-    outputs=['aws-firehose:alerts'],
     req_subkeys={
         'detail': ['requestParameters']
     })

diff --git a/rules/community/cloudtrail/cloudtrail_root_account_usage.py b/rules/community/cloudtrail/cloudtrail_root_account_usage.py
@@ -6,7 +6,6 @@
 
 @rule(
     logs=['cloudwatch:events'],
-    outputs=['aws-firehose:alerts'],
     req_subkeys={
         'detail': ['userIdentity', 'eventType']
     })

diff --git a/rules/community/cloudtrail/cloudtrail_security_group_ingress_anywhere.py b/rules/community/cloudtrail/cloudtrail_security_group_ingress_anywhere.py
@@ -7,7 +7,6 @@
 
 @rule(
     logs=['cloudwatch:events'],
-    outputs=['aws-firehose:alerts'],
     req_subkeys={
         'detail': ['eventName', 'requestParameters']
     })

diff --git a/rules/community/duo_administrator/duo_bypass_code_create_non_auto_generated.py b/rules/community/duo_administrator/duo_bypass_code_create_non_auto_generated.py
@@ -5,7 +5,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['duo:administrator'], outputs=['aws-firehose:alerts'])
+@rule(logs=['duo:administrator'])
 def duo_bypass_code_create_non_auto_generated(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/duo_administrator/duo_bypass_code_create_non_expiring.py b/rules/community/duo_administrator/duo_bypass_code_create_non_expiring.py
@@ -5,7 +5,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['duo:administrator'], outputs=['aws-firehose:alerts'])
+@rule(logs=['duo:administrator'])
 def duo_bypass_code_create_non_expiring(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/duo_administrator/duo_bypass_code_create_unlimited_use.py b/rules/community/duo_administrator/duo_bypass_code_create_unlimited_use.py
@@ -5,7 +5,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['duo:administrator'], outputs=['aws-firehose:alerts'])
+@rule(logs=['duo:administrator'])
 def duo_bypass_code_create_unlimited_use(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/duo_authentication/duo_anonymous_ip_failure.py b/rules/community/duo_authentication/duo_anonymous_ip_failure.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['duo:authentication'], outputs=['aws-firehose:alerts'])
+@rule(logs=['duo:authentication'])
 def duo_anonymous_ip_failure(rec):
     """
     author:       airbnb_csirt

diff --git a/rules/community/duo_authentication/duo_fraud.py b/rules/community/duo_authentication/duo_fraud.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['duo:authentication'], outputs=['aws-firehose:alerts'])
+@rule(logs=['duo:authentication'])
 def duo_fraud(rec):
     """
     author:       airbnb_csirt

diff --git a/rules/community/github/github_disable_dismiss_stale_pull_request_approvals.py b/rules/community/github/github_disable_dismiss_stale_pull_request_approvals.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_dismiss_stale_pull_request_approvals(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_disable_protect_this_branch.py b/rules/community/github/github_disable_protect_this_branch.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_protect_this_branch(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_disable_required_pull_request_reviews.py b/rules/community/github/github_disable_required_pull_request_reviews.py
@@ -5,7 +5,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_required_pull_request_reviews(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_disable_required_status_checks.py b/rules/community/github/github_disable_required_status_checks.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_required_status_checks(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_disable_two_factor_requirement_org.py b/rules/community/github/github_disable_two_factor_requirement_org.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_two_factor_requirement_org(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_disable_two_factor_requirement_user.py b/rules/community/github/github_disable_two_factor_requirement_user.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_disable_two_factor_requirement_user(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_oauth_application_create.py b/rules/community/github/github_oauth_application_create.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_oauth_application_create(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_site_admin_action.py b/rules/community/github/github_site_admin_action.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_site_admin_action(rec):
     """
     author:       @mimeframe

diff --git a/rules/community/github/github_site_admin_user_promotion.py b/rules/community/github/github_site_admin_user_promotion.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['ghe:general'], outputs=['aws-firehose:alerts'])
+@rule(logs=['ghe:general'])
 def github_site_admin_user_promotion(rec):
     """
     author:       @fusionrace, @mimeframe

diff --git a/rules/community/guardduty/guard_duty_all.py b/rules/community/guardduty/guard_duty_all.py
@@ -5,7 +5,7 @@
 disable = StreamRules.disable()
 
 
-@rule(logs=['cloudwatch:events'], matchers=['guard_duty'], outputs=['aws-firehose:alerts'])
+@rule(logs=['cloudwatch:events'], matchers=['guard_duty'])
 def guard_duty_all(*_):
     """
     author:         spiper

diff --git a/...e_attack/defense_evasion/multi/obfuscated_files_or_information/right_to_left_character.py b/...e_attack/defense_evasion/multi/obfuscated_files_or_information/right_to_left_character.py
@@ -5,9 +5,7 @@
 rule = StreamRules.rule
 
 
-@rule(
-    datatypes=['command', 'filePath', 'processPath', 'fileName'],
-    outputs=['aws-firehose:alerts'])
+@rule(datatypes=['command', 'filePath', 'processPath', 'fileName'])
 def right_to_left_character(rec):
     """
     author:           @javutin

diff --git a/rules/community/onelogin/onelogin_events_assumed_role.py b/rules/community/onelogin/onelogin_events_assumed_role.py
@@ -4,7 +4,7 @@
 rule = StreamRules.rule
 
 
-@rule(logs=['onelogin:events'], outputs=['aws-firehose:alerts'])
+@rule(logs=['onelogin:events'])
 def onelogin_events_assumed_role(rec):
     """
     author:       @javutin

diff --git a/stream_alert/alert_processor/main.py b/stream_alert/alert_processor/main.py
@@ -20,7 +20,7 @@
 from stream_alert.alert_processor import LOGGER
 from stream_alert.alert_processor.helpers import validate_alert
 from stream_alert.alert_processor.outputs.output_base import StreamAlertOutput
-from stream_alert.shared import NORMALIZATION_KEY
+from stream_alert.shared import NORMALIZATION_KEY, resources
 
 
 def handler(event, context):
@@ -41,14 +41,15 @@ def handler(event, context):
     """
     # A failure to load the config will log the error in load_output_config
     # and return here
-    config = _load_output_config()
-    if not config:
-        return
-
     split_arn = context.invoked_function_arn.split(':')
     region = split_arn[3]
     account_id = split_arn[4]
     function_name = context.function_name
+    prefix = function_name.split('_')[0]
+
+    config = _load_output_config(prefix)
+    if not config:
+        return
 
     # Return the current list of statuses back to the caller
     return list(run(event, region, account_id, function_name, config))
@@ -151,7 +152,7 @@ def _sort_dict(unordered_dict):
     return result
 
 
-def _load_output_config(config_path='conf/outputs.json'):
+def _load_output_config(prefix, config_path='conf/outputs.json'):
     """Load the outputs configuration file from disk
 
     Returns:
@@ -164,4 +165,7 @@ def _load_output_config(config_path='conf/outputs.json'):
             LOGGER.error('The \'%s\' file could not be loaded into json', config_path)
             return
 
+    # Merge in the default outputs info
+    config = resources.merge_required_outputs(config, prefix)
+
     return config
diff --git a/stream_alert/rule_processor/rules_engine.py b/stream_alert/rule_processor/rules_engine.py
@@ -19,7 +19,7 @@
 
 from stream_alert.rule_processor import LOGGER
 from stream_alert.rule_processor.threat_intel import StreamThreatIntel
-from stream_alert.shared import NORMALIZATION_KEY
+from stream_alert.shared import NORMALIZATION_KEY, resources
 
 DEFAULT_RULE_DESCRIPTION = 'No rule description provided'
 
@@ -53,6 +53,7 @@ class StreamRules(object):
     def __init__(self, config):
         """Initialize a StreamRules instance to cache a StreamThreatIntel instance."""
         self._threat_intel = StreamThreatIntel.load_from_config(config)
+        self._required_outputs_set = resources.get_required_outputs()
 
     @classmethod
     def get_rules(cls):
@@ -85,12 +86,6 @@ def decorator(rule):
                     rule_name)
                 return
 
-            if not outputs:
-                LOGGER.error(
-                    'Invalid rule [%s] - rule must have \'outputs\' declared',
-                    rule_name)
-                return
-
             if rule_name in cls.__rules:
                 raise ValueError('rule [{}] already defined'.format(rule_name))
             cls.__rules[rule_name] = RuleAttributes(rule_name,
@@ -397,6 +392,8 @@ def process(self, input_payload):
 
         return alerts, normalized_records
 
+
+
     def threat_intel_match(self, payload_with_normalized_records):
         """Apply Threat Intelligence on normalized records
 
@@ -421,8 +418,7 @@ def threat_intel_match(self, payload_with_normalized_records):
                         self.rule_analysis(ioc_record.pre_parsed_record, rule, ioc_record, alerts)
         return alerts
 
-    @staticmethod
-    def rule_analysis(record, rule, payload, alerts):
+    def rule_analysis(self, record, rule, payload, alerts):
         """Class method to analyze rule against a record
 
         Args:
@@ -443,14 +439,18 @@ def rule_analysis(record, rule, payload, alerts):
             LOGGER.info('Rule [%s] triggered alert [%s] on log type [%s] from entity \'%s\' '
                         'in service \'%s\'', rule.rule_name, alert_id, payload.log_source,
                         payload.entity, payload.service())
+
+            # Combine the required alert outputs with the ones for this rule
+            all_outputs = self._required_outputs_set.union(set(rule.outputs or []))
+
             alert = {
                 'id': alert_id,
                 'record': record,
                 'rule_name': rule.rule_name,
                 'rule_description': rule.rule_function.__doc__ or DEFAULT_RULE_DESCRIPTION,
                 'log_source': str(payload.log_source),
                 'log_type': payload.type,
-                'outputs': rule.outputs,
+                'outputs': list(all_outputs), # TODO: @austinbyers - change this to a set
                 'source_service': payload.service(),
                 'source_entity': payload.entity,
                 'context': rule.context}