From ae5408e4d26a96daa5b71c548966eb4703dbfbba Mon Sep 17 00:00:00 2001 From: javier_marcos Date: Mon, 20 Nov 2017 09:56:36 -0800 Subject: [PATCH 1/3] Header from is required to create incident --- stream_alert/alert_processor/outputs.py | 74 ++++++++----- .../test_outputs.py | 103 +++++++++++++----- 2 files changed, 119 insertions(+), 58 deletions(-) diff --git a/stream_alert/alert_processor/outputs.py b/stream_alert/alert_processor/outputs.py index 496e964b4..dc5208f2b 100644 --- a/stream_alert/alert_processor/outputs.py +++ b/stream_alert/alert_processor/outputs.py @@ -249,7 +249,11 @@ def get_user_defined_properties(self): mask_input=True, cred_requirement=True)), ('escalation_policy', - OutputProperty(description='the name of the default escalation policy')) + OutputProperty(description='the name of the default escalation policy')), + ('email_from', + OutputProperty(description='valid user email from the PagerDuty ' + 'account linked to the token', + cred_requirement=True)) ]) @staticmethod @@ -265,7 +269,7 @@ def _get_endpoint(base_url, endpoint): """ return os.path.join(base_url, endpoint) - def _check_exists_get_id(self, filter_str, url, target_key): + def _check_exists(self, filter_str, url, target_key, get_id=True): """Generic method to run a search in the PagerDuty REST API and return the id of the first occurence from the results. @@ -273,10 +277,11 @@ def _check_exists_get_id(self, filter_str, url, target_key): filter_str (str): The query filter to search for in the API url (str): The url to send the requests to in the API target_key (str): The key to extract in the returned results + get_id (boolean): Whether to generate a dict with result and reference Returns: str: ID of the targeted element that matches the provided filter or - False if a matching element does not exists. + True/False whether a matching element exists or not. """ params = { 'query': '{}'.format(filter_str) @@ -291,21 +296,23 @@ def _check_exists_get_id(self, filter_str, url, target_key): return False # If there are results, get the first occurence from the list - return response[target_key][0]['id'] if target_key in response else False + if get_id: + return response[target_key][0]['id'] if target_key in response else False + + return True - def _user_verify(self, user): + def _user_verify(self, user, get_id=True): """Method to verify the existance of an user with the API Args: user (str): User to query about in the API. + get_id (boolean): Whether to generate a dict with result and reference Returns: dict or False: JSON object be used in the API call, containing the user_id and user_reference. False if user is not found """ - users_url = self._get_endpoint(self._base_url, self.USERS_ENDPOINT) - - return self._item_verify(users_url, user, self.USERS_ENDPOINT, 'user_reference') + return self._item_verify(user, self.USERS_ENDPOINT, 'user_reference', get_id) def _policy_verify(self, policy, default_policy): """Method to verify the existance of a escalation policy with the API @@ -318,16 +325,13 @@ def _policy_verify(self, policy, default_policy): dict: JSON object be used in the API call, containing the policy_id and escalation_policy_reference """ - policies_url = self._get_endpoint(self._base_url, self.POLICIES_ENDPOINT) - - verified = self._item_verify(policies_url, policy, self.POLICIES_ENDPOINT, - 'escalation_policy_reference') + verified = self._item_verify(policy, self.POLICIES_ENDPOINT, 'escalation_policy_reference') # If the escalation policy provided is not verified in the API, use the default if verified: return verified - return self._item_verify(policies_url, default_policy, self.POLICIES_ENDPOINT, + return self._item_verify(default_policy, self.POLICIES_ENDPOINT, 'escalation_policy_reference') def _service_verify(self, service): @@ -340,32 +344,34 @@ def _service_verify(self, service): dict: JSON object be used in the API call, containing the service_id and the service_reference """ - services_url = self._get_endpoint(self._base_url, self.SERVICES_ENDPOINT) + return self._item_verify(service, self.SERVICES_ENDPOINT, 'service_reference') - return self._item_verify(services_url, service, self.SERVICES_ENDPOINT, 'service_reference') - - def _item_verify(self, item_url, item_str, item_key, item_type): + def _item_verify(self, item_str, item_key, item_type, get_id=True): """Method to verify the existance of an item with the API Args: - item_url (str): URL of the API Endpoint within the API to query item_str (str): Service to query about in the API - item_key (str): Key to be extracted from search results + item_key (str): Endpoint/key to be extracted from search results item_type (str): Type of item reference to be returned + get_id (boolean): Whether to generate a dict with result and reference Returns: dict: JSON object be used in the API call, containing the item id - and the item reference, or False if it fails + and the item reference, True if it just exists or False if it fails """ - item_id = self._check_exists_get_id(item_str, item_url, item_key) + item_url = self._get_endpoint(self._base_url, item_key) + item_id = self._check_exists(item_str, item_url, item_key, get_id) if not item_id: LOGGER.info('%s not found in %s, %s', item_str, item_key, self.__service__) return False - return { - 'id': item_id, - 'type': item_type - } + if get_id: + return { + 'id': item_id, + 'type': item_type + } + + return item_id def _incident_assignment(self, context): """Method to determine if the incident gets assigned to a user or an escalation policy @@ -407,14 +413,22 @@ def dispatch(self, **kwargs): if not creds: return self._log_status(False) + # Cache base_url + self._base_url = creds['api'] + # Preparing headers for API calls self._headers = { 'Authorization': 'Token token={}'.format(creds['token']), 'Accept': 'application/vnd.pagerduty+json;version=2' } - # Cache base_url - self._base_url = creds['api'] + # Get user email to be added as From header and verify + user_email = creds['email_from'] + if not self._user_verify(user_email, False): + return self._log_status(False) + + # Add From to the headers after verifying + self._headers['From'] = user_email # Cache default escalation policy self._escalation_policy = creds['escalation_policy'] @@ -441,9 +455,9 @@ def dispatch(self, **kwargs): 'type': 'incident', 'title': incident_title, 'service': incident_service, - 'body': incident_body - }, - assigned_key: assigned_value + 'body': incident_body, + assigned_key: assigned_value + } } incidents_url = self._get_endpoint(self._base_url, self.INCIDENTS_ENDPOINT) resp = self._post_request(incidents_url, incident, self._headers, True) diff --git a/tests/unit/stream_alert_alert_processor/test_outputs.py b/tests/unit/stream_alert_alert_processor/test_outputs.py index 1b9aa0f67..e59d6bbb0 100644 --- a/tests/unit/stream_alert_alert_processor/test_outputs.py +++ b/tests/unit/stream_alert_alert_processor/test_outputs.py @@ -314,7 +314,8 @@ def _setup_dispatch(self, context=None): creds = {'api': 'https://api.pagerduty.com', 'token': 'mocked_token', 'service_key': 'mocked_service_key', - 'escalation_policy': 'mocked_escalation_policy'} + 'escalation_policy': 'mocked_escalation_policy', + 'email_from': 'email@domain.com'} put_mock_creds(output_name, creds, self.__dispatcher.secrets_bucket, REGION, KMS_ALIAS) @@ -332,7 +333,7 @@ def test_check_exists_get_id(self, get_mock): json_check = json.loads('{"check": [{"id": "checked_id"}]}') get_mock.return_value.json.return_value = json_check - checked = self.__dispatcher._check_exists_get_id('filter', 'http://mock_url', 'check') + checked = self.__dispatcher._check_exists('filter', 'http://mock_url', 'check') assert_equal(checked, 'checked_id') @patch('requests.get') @@ -343,9 +344,19 @@ def test_check_exists_get_id_fail(self, get_mock): json_check = json.loads('{}') get_mock.return_value.json.return_value = json_check - checked = self.__dispatcher._check_exists_get_id('filter', 'http://mock_url', 'check') + checked = self.__dispatcher._check_exists('filter', 'http://mock_url', 'check') assert_false(checked) + @patch('requests.get') + def test_check_exists_no_get_id(self, get_mock): + """Check Exists No Get Id - PagerDutyIncidentOutput""" + # /check + get_mock.return_value.status_code = 200 + json_check = json.loads('{"check": [{"id": "checked_id"}]}') + get_mock.return_value.json.return_value = json_check + + assert_true(self.__dispatcher._check_exists('filter', 'http://mock_url', 'check', False)) + @patch('requests.get') def test_user_verify_success(self, get_mock): """User Verify Success - PagerDutyIncidentOutput""" @@ -444,11 +455,20 @@ def test_item_verify_success(self, get_mock): json_check = json.loads('{"items": [{"id": "verified_item_id"}]}') get_mock.return_value.json.return_value = json_check - item_verified = self.__dispatcher._item_verify('http://mock_url', 'valid_item', - 'items', 'item_reference') + item_verified = self.__dispatcher._item_verify('valid_item', 'items', 'item_reference') assert_equal(item_verified['id'], 'verified_item_id') assert_equal(item_verified['type'], 'item_reference') + @patch('requests.get') + def test_item_verify_no_get_id_success(self, get_mock): + """Item Verify No Get Id Success - PagerDutyIncidentOutput""" + # /items + get_mock.return_value.status_code = 200 + json_check = json.loads('{"items": [{"id": "verified_item_id"}]}') + get_mock.return_value.json.return_value = json_check + + assert_true(self.__dispatcher._item_verify('valid_item', 'items', 'item_reference', False)) + @patch('requests.get') def test_incident_assignment_user(self, get_mock): """Incident Assignment User - PagerDutyIncidentOutput""" @@ -518,11 +538,11 @@ def test_dispatch_success_good_user(self, get_mock, post_mock, log_info_mock): } alert = self._setup_dispatch(context=ctx) - # /users, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200]) + # /users, /users, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200]) json_user = json.loads('{"users": [{"id": "valid_user_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_user, json_service] + get_mock.return_value.json.side_effect = [json_user, json_user, json_service] # /incidents post_mock.return_value.status_code = 200 @@ -549,11 +569,12 @@ def test_dispatch_success_good_policy(self, get_mock, post_mock, log_info_mock): } alert = self._setup_dispatch(context=ctx) - # /escalation_policies, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200]) + # /users, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') json_policy = json.loads('{"escalation_policies": [{"id": "policy_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_policy, json_service] + get_mock.return_value.json.side_effect = [json_user, json_policy, json_service] # /incidents post_mock.return_value.status_code = 200 @@ -580,12 +601,14 @@ def test_dispatch_success_bad_user(self, get_mock, post_mock, log_info_mock): } alert = self._setup_dispatch(context=ctx) - # /users, /escalation_policies, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200]) - json_user = json.loads('{"not_users": [{"id": "user_id"}]}') + # /users, /users, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200, 200]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') + json_not_user = json.loads('{"not_users": [{"id": "user_id"}]}') json_policy = json.loads('{"escalation_policies": [{"id": "policy_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_user, json_policy, json_service] + get_mock.return_value.json.side_effect = [json_user, json_not_user, + json_policy, json_service] # /incidents post_mock.return_value.status_code = 200 @@ -607,11 +630,12 @@ def test_dispatch_success_no_context(self, get_mock, post_mock, log_info_mock): """PagerDutyIncidentOutput dispatch success - No Context""" alert = self._setup_dispatch() - # /escalation_policies, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200]) + # /users, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') json_policy = json.loads('{"escalation_policies": [{"id": "policy_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_policy, json_service] + get_mock.return_value.json.side_effect = [json_user, json_policy, json_service] # /incidents post_mock.return_value.status_code = 200 @@ -632,10 +656,11 @@ def test_dispatch_success_no_context(self, get_mock, post_mock, log_info_mock): def test_dispatch_failure_bad_everything(self, get_mock, post_mock, log_error_mock): """PagerDutyIncidentOutput dispatch failure - No User, Bad Policy, Bad Service""" alert = self._setup_dispatch() - # /escalation_policies, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[400, 400, 400]) + # /users, /users, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 400, 400, 400]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') json_empty = json.loads('{}') - get_mock.return_value.json.side_effect = [json_empty, json_empty, json_empty] + get_mock.return_value.json.side_effect = [json_user, json_empty, json_empty, json_empty] # /incidents post_mock.return_value.status_code = 400 @@ -661,13 +686,14 @@ def test_dispatch_success_bad_policy(self, get_mock, post_mock, log_info_mock): } } alert = self._setup_dispatch(context=ctx) - # /escalation_policies, /services - get_mock.return_value.side_effect = [400, 200, 200] - type(get_mock.return_value).status_code = PropertyMock(side_effect=[400, 200, 200]) + # /users, /escalation_policies, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 400, 200, 200]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') json_bad_policy = json.loads('{}') json_good_policy = json.loads('{"escalation_policies": [{"id": "policy_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_bad_policy, json_good_policy, json_service] + get_mock.return_value.json.side_effect = [json_user, json_bad_policy, + json_good_policy, json_service] # /incidents post_mock.return_value.status_code = 200 @@ -688,11 +714,12 @@ def test_dispatch_success_bad_policy(self, get_mock, post_mock, log_info_mock): def test_dispatch_bad_dispatch(self, get_mock, post_mock, log_error_mock): """PagerDutyIncidentOutput dispatch - Bad Dispatch""" alert = self._setup_dispatch() - # /escalation_policies, /services - type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200]) + # /users, /escalation_policies, /services + type(get_mock.return_value).status_code = PropertyMock(side_effect=[200, 200, 200]) + json_user = json.loads('{"users": [{"id": "user_id"}]}') json_policy = json.loads('{"escalation_policies": [{"id": "policy_id"}]}') json_service = json.loads('{"services": [{"id": "service_id"}]}') - get_mock.return_value.json.side_effect = [json_policy, json_service] + get_mock.return_value.json.side_effect = [json_user, json_policy, json_service] # /incidents post_mock.return_value.status_code = 400 @@ -705,6 +732,26 @@ def test_dispatch_bad_dispatch(self, get_mock, post_mock, log_error_mock): log_error_mock.assert_called_with('Failed to send alert to %s', self.__service) + @patch('logging.Logger.error') + @patch('requests.get') + @mock_s3 + @mock_kms + def test_dispatch_bad_email(self, get_mock, log_error_mock): + """PagerDutyIncidentOutput dispatch - Bad Email""" + alert = self._setup_dispatch() + # /users, /escalation_policies, /services + get_mock.return_value.status_code = 400 + json_user = json.loads('{"not_users": [{"id": "no_user_id"}]}') + get_mock.return_value.json.return_value = json_user + + self.__dispatcher.dispatch(descriptor=self.__descriptor, + rule_name='rule_name', + alert=alert) + + self._teardown_dispatch() + + log_error_mock.assert_called_with('Failed to send alert to %s', self.__service) + @patch('logging.Logger.error') @mock_s3 @mock_kms From 2c7744e5a093cde5c77caa3b820cf8ae0883dc21 Mon Sep 17 00:00:00 2001 From: javier_marcos Date: Mon, 20 Nov 2017 13:39:05 -0800 Subject: [PATCH 2/3] Comments addressed --- stream_alert/alert_processor/outputs.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/stream_alert/alert_processor/outputs.py b/stream_alert/alert_processor/outputs.py index dc5208f2b..a3c312912 100644 --- a/stream_alert/alert_processor/outputs.py +++ b/stream_alert/alert_processor/outputs.py @@ -295,11 +295,11 @@ def _check_exists(self, filter_str, url, target_key, get_id=True): if not response: return False - # If there are results, get the first occurence from the list - if get_id: - return response[target_key][0]['id'] if target_key in response else False + if not get_id: + return True - return True + # If there are results, get the first occurence from the list + return response[target_key][0]['id'] if target_key in response else False def _user_verify(self, user, get_id=True): """Method to verify the existance of an user with the API @@ -366,10 +366,7 @@ def _item_verify(self, item_str, item_key, item_type, get_id=True): return False if get_id: - return { - 'id': item_id, - 'type': item_type - } + return {'id': item_id, 'type': item_type} return item_id From 6406c948b5678d4ec87fe3da9a98ac242239b666 Mon Sep 17 00:00:00 2001 From: javier_marcos Date: Mon, 20 Nov 2017 13:50:58 -0800 Subject: [PATCH 3/3] Logging when email for header From can not be verified --- stream_alert/alert_processor/outputs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stream_alert/alert_processor/outputs.py b/stream_alert/alert_processor/outputs.py index a3c312912..96baaaad5 100644 --- a/stream_alert/alert_processor/outputs.py +++ b/stream_alert/alert_processor/outputs.py @@ -422,6 +422,7 @@ def dispatch(self, **kwargs): # Get user email to be added as From header and verify user_email = creds['email_from'] if not self._user_verify(user_email, False): + LOGGER.error('Could not verify header From: %s, %s', user_email, self.__service__) return self._log_status(False) # Add From to the headers after verifying