From bd5d5bffc25709e46e51d11307081a8d69aeca8f Mon Sep 17 00:00:00 2001 From: Jason Conger Date: Wed, 13 Jul 2022 18:02:44 -0500 Subject: [PATCH] fix: bug fix for pagination issue --- CHANGELOG.md | 4 ++++ README.md | 2 +- globalConfig.json | 2 +- package/app.manifest | 2 +- package/bin/MS_AAD_audit.py | 5 +++-- package/bin/MS_AAD_device.py | 5 +++-- package/bin/MS_AAD_group.py | 5 +++-- package/bin/MS_AAD_identity_protection.py | 10 ++++++---- package/bin/MS_AAD_signins.py | 5 +++-- package/bin/MS_AAD_user.py | 5 +++-- package/bin/azure_comp.py | 20 +++++++++++-------- .../bin/azure_reservation_recommendation.py | 5 +++-- package/bin/azure_resource_group.py | 5 +++-- package/bin/azure_security_center_input.py | 10 ++++++---- package/bin/azure_subscription.py | 5 +++-- package/bin/azure_virtual_network.py | 20 +++++++++++-------- package/bin/ta_azure_utils/utils.py | 6 +++--- 17 files changed, 70 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1168d9..209524e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Version 4.0.1 +* Bug fix - Problem creating new AAD Audit Input - [Issue #3](https://github.com/splunk/splunk-add-on-microsoft-azure/issues/3) +* Bug fix - Azure AD User and Group pagination issue + # Version 4.0.0 * **BREAKING CHANGE**: removed deprecated event hub input. Use the [Splunk Add-on for Microsoft Cloud Services](https://splunkbase.splunk.com/app/3110/) to collect event hub data. * Code is now open source [https://github.com/splunk/splunk-add-on-microsoft-azure](https://github.com/splunk/splunk-add-on-microsoft-azure) diff --git a/README.md b/README.md index d986205..bb827d2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This add-on is built with Splunk's [UCC Generator](https://github.com/splunk/add Example: - ucc-gen --ta-version=4.0.0 + ucc-gen --ta-version=4.0.1 The add-on will be built in an `output` directory in the root of the repository. diff --git a/globalConfig.json b/globalConfig.json index 73f86fe..3f57afa 100644 --- a/globalConfig.json +++ b/globalConfig.json @@ -2,7 +2,7 @@ "meta": { "name": "TA-MS-AAD", "displayName": "Splunk Add-on for Microsoft Azure", - "version": "4.0.0", + "version": "4.0.1", "apiVersion": "3.0.0", "restRoot": "TA_MS_AAD", "schemaVersion": "0.0.3" diff --git a/package/app.manifest b/package/app.manifest index 24b5cde..04e0646 100644 --- a/package/app.manifest +++ b/package/app.manifest @@ -5,7 +5,7 @@ "id": { "group": null, "name": "TA-MS-AAD", - "version": "4.0.0" + "version": "4.0.1" }, "author": [ { diff --git a/package/bin/MS_AAD_audit.py b/package/bin/MS_AAD_audit.py index 84978e6..60a4919 100755 --- a/package/bin/MS_AAD_audit.py +++ b/package/bin/MS_AAD_audit.py @@ -162,7 +162,7 @@ def collect_events(helper, ew): max_activityDate = query_date response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -184,7 +184,8 @@ def collect_events(helper, ew): # Check point the largest activityDate seen during the query helper.save_check_point(check_point_key, max_activityDate) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: helper.log_error("_Splunk_ Unable to obtain access token") diff --git a/package/bin/MS_AAD_device.py b/package/bin/MS_AAD_device.py index 405a3cb..54b4d30 100755 --- a/package/bin/MS_AAD_device.py +++ b/package/bin/MS_AAD_device.py @@ -104,7 +104,7 @@ def collect_events(helper, ew): url = graph_base_url + "/%s/devices" % endpoint response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -116,7 +116,8 @@ def collect_events(helper, ew): ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: helper.log_error("_Splunk_ Unable to obtain access token") diff --git a/package/bin/MS_AAD_group.py b/package/bin/MS_AAD_group.py index 3e2571e..2132203 100755 --- a/package/bin/MS_AAD_group.py +++ b/package/bin/MS_AAD_group.py @@ -110,7 +110,7 @@ def collect_events(helper, ew): url = "%s?%s" % (url, filter) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -122,7 +122,8 @@ def collect_events(helper, ew): ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: helper.log_error("_Splunk_ Unable to obtain access token") diff --git a/package/bin/MS_AAD_identity_protection.py b/package/bin/MS_AAD_identity_protection.py index 4ac20eb..88ce7eb 100755 --- a/package/bin/MS_AAD_identity_protection.py +++ b/package/bin/MS_AAD_identity_protection.py @@ -133,7 +133,7 @@ def collect_events(helper, ew): url = graph_base_url + "/%s/identityProtection/riskDetections?$orderby=lastUpdatedDateTime&$filter=lastUpdatedDateTime gt %s" % (endpoint, risk_detection_check_point) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] max_risk_detection_date = risk_detection_check_point while items: for item in items: @@ -152,7 +152,8 @@ def collect_events(helper, ew): sys.stdout.flush() helper.save_check_point(risk_detection_check_point_key, max_risk_detection_date) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_risky_user_data): @@ -167,7 +168,7 @@ def collect_events(helper, ew): url = graph_base_url + "/%s/identityProtection/riskyUsers?$orderby=riskLastUpdatedDateTime&$filter=riskLastUpdatedDateTime gt %s" % (endpoint, risky_user_check_point) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] max_risky_user_date = risky_user_check_point while items: for item in items: @@ -186,7 +187,8 @@ def collect_events(helper, ew): sys.stdout.flush() helper.save_check_point(risky_user_check_point_key, max_risky_user_date) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/MS_AAD_signins.py b/package/bin/MS_AAD_signins.py index cd9b16a..c54cdce 100755 --- a/package/bin/MS_AAD_signins.py +++ b/package/bin/MS_AAD_signins.py @@ -184,7 +184,7 @@ def collect_events(helper, ew): max_signinDateTime = query_date response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -206,7 +206,8 @@ def collect_events(helper, ew): # Check point the largest signinDateTime seen during the query helper.save_check_point(check_point_key, max_signinDateTime) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: helper.log_error("_Splunk_ Unable to obtain access token") diff --git a/package/bin/MS_AAD_user.py b/package/bin/MS_AAD_user.py index 309863e..9faadd8 100755 --- a/package/bin/MS_AAD_user.py +++ b/package/bin/MS_AAD_user.py @@ -109,7 +109,7 @@ def collect_events(helper, ew): url = "%s?%s" % (url, filter) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -121,7 +121,8 @@ def collect_events(helper, ew): ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: helper.log_error("_Splunk_ Unable to obtain access token") diff --git a/package/bin/azure_comp.py b/package/bin/azure_comp.py index 5579fff..3fb1637 100755 --- a/package/bin/azure_comp.py +++ b/package/bin/azure_comp.py @@ -147,7 +147,7 @@ def collect_events(helper, ew): helper.log_debug("_Splunk_ input_name=%s Collecting managed disk data. sourcetype='%s'" % (input_name, disk_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Compute/disks?api-version=%s" % (subscription_id, disk_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -157,14 +157,15 @@ def collect_events(helper, ew): sourcetype=disk_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_images): helper.log_debug("_Splunk_ input_name=%s Collecting image data. sourcetype='%s'" % (input_name, image_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Compute/images?api-version=%s" % (subscription_id, image_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -174,13 +175,14 @@ def collect_events(helper, ew): sourcetype=image_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_snapshots): helper.log_debug("_Splunk_ input_name=%s Collecting snapshot data. sourcetype='%s'" % (input_name, snapshot_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Compute/snapshots?api-version=%s" % (subscription_id, snapshot_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -190,13 +192,14 @@ def collect_events(helper, ew): sourcetype=snapshot_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_vms): helper.log_debug("_Splunk_ input_name=%s Collecting virtual machine data. sourcetype='%s'" % (input_name, vm_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Compute/virtualMachines?api-version=%s" % (subscription_id, vm_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: try: @@ -228,7 +231,8 @@ def collect_events(helper, ew): sourcetype=vm_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/azure_reservation_recommendation.py b/package/bin/azure_reservation_recommendation.py index 7065e93..ce51935 100755 --- a/package/bin/azure_reservation_recommendation.py +++ b/package/bin/azure_reservation_recommendation.py @@ -103,7 +103,7 @@ def collect_events(helper, ew): helper.log_debug("_Splunk_ input_name=%s Collecting reservation recommendation data." % input_name) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Consumption/reservationRecommendations?api-version=2019-05-01" % subscription_id response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -114,7 +114,8 @@ def collect_events(helper, ew): ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/azure_resource_group.py b/package/bin/azure_resource_group.py index fee83ab..cfbb403 100755 --- a/package/bin/azure_resource_group.py +++ b/package/bin/azure_resource_group.py @@ -102,7 +102,7 @@ def collect_events(helper, ew): helper.log_debug("_Splunk_ input_name=%s Collecting resource group data." % input_name) url = management_base_url + "/subscriptions/%s/resourcegroups?api-version=%s" % (subscription_id, resource_group_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: @@ -115,7 +115,8 @@ def collect_events(helper, ew): sys.stdout.flush() sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/azure_security_center_input.py b/package/bin/azure_security_center_input.py index e3835c5..ffb84ae 100755 --- a/package/bin/azure_security_center_input.py +++ b/package/bin/azure_security_center_input.py @@ -134,7 +134,7 @@ def collect_events(helper, ew): url = management_base_url + "/subscriptions/%s/providers/Microsoft.Security/alerts?api-version=%s&$filter=Properties/DetectedTimeUtc gt %s" % (subscription_id, alert_api_version, alert_check_point) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] max_asc_alert_date = alert_check_point while items: @@ -154,7 +154,8 @@ def collect_events(helper, ew): sys.stdout.flush() helper.save_check_point(alert_check_point_key, max_asc_alert_date) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_tasks): helper.log_debug("_Splunk_ input_name=%s Collecting security task data. sourcetype='%s'" % (input_name, task_sourcetype)) @@ -167,7 +168,7 @@ def collect_events(helper, ew): url = management_base_url + "/subscriptions/%s/providers/Microsoft.Security/tasks?api-version=%s&$filter=Properties/LastStateChangeTimeUtc gt %s" % (subscription_id, task_api_version, task_check_point) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] max_asc_task_date = task_check_point while items: @@ -186,7 +187,8 @@ def collect_events(helper, ew): sys.stdout.flush() helper.save_check_point(task_check_point_key, max_asc_task_date) - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/azure_subscription.py b/package/bin/azure_subscription.py index 72b480f..8e80b5e 100755 --- a/package/bin/azure_subscription.py +++ b/package/bin/azure_subscription.py @@ -99,7 +99,7 @@ def collect_events(helper, ew): helper.log_debug("_Splunk_ input_name=%s Collecting subscription data." % input_name) url = management_base_url + "/subscriptions?api-version=%s" % api_version response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -109,7 +109,8 @@ def collect_events(helper, ew): sourcetype=source_type) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/azure_virtual_network.py b/package/bin/azure_virtual_network.py index eb18a2b..6dc197e 100755 --- a/package/bin/azure_virtual_network.py +++ b/package/bin/azure_virtual_network.py @@ -144,7 +144,7 @@ def collect_events(helper, ew): helper.log_debug("_Splunk_ input_name=%s Collecting virtual network data. sourcetype='%s'" % (input_name, vnet_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Network/virtualNetworks?api-version=%s" % (subscription_id, vnet_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -154,13 +154,14 @@ def collect_events(helper, ew): sourcetype=vnet_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_nics): helper.log_debug("_Splunk_ input_name=%s Collecting nic data. sourcetype='%s'" % (input_name, nic_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Network/networkInterfaces?api-version=%s" % (subscription_id, nic_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -170,13 +171,14 @@ def collect_events(helper, ew): sourcetype=nic_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_nsgs): helper.log_debug("_Splunk_ input_name=%s Collecting nsg data. sourcetype='%s'" % (input_name, nsg_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Network/networkSecurityGroups?api-version=%s" % (subscription_id, nsg_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -186,13 +188,14 @@ def collect_events(helper, ew): sourcetype=nsg_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] if(collect_ips): helper.log_debug("_Splunk_ input_name=%s Collecting IP address data. sourcetype='%s'" % (input_name, ip_sourcetype)) url = management_base_url + "/subscriptions/%s/providers/Microsoft.Network/publicIPAddresses?api-version=%s" % (subscription_id, ip_api_version) response = azutils.get_items_batch_session(helper=helper, url=url, session=session) - items = response['value'] or None + items = None if response == None else response['value'] while items: for item in items: event = helper.new_event( @@ -202,7 +205,8 @@ def collect_events(helper, ew): sourcetype=ip_sourcetype) ew.write_event(event) sys.stdout.flush() - items = azutils.handle_nextLink(helper=helper, response=response, session=session) + response = azutils.handle_nextLink(helper=helper, response=response, session=session) + items = None if response == None else response['value'] else: raise RuntimeError("Unable to obtain access token. Please check the Client ID, Client Secret, and Tenant ID") diff --git a/package/bin/ta_azure_utils/utils.py b/package/bin/ta_azure_utils/utils.py index 1ca2c69..babaab7 100755 --- a/package/bin/ta_azure_utils/utils.py +++ b/package/bin/ta_azure_utils/utils.py @@ -29,7 +29,6 @@ TIMEOUT = 5 #seconds def handle_nextLink(helper=None, response=None, session=None): - items = None if '@odata.nextLink' in response: nextLink = response['@odata.nextLink'] helper.log_debug("_Splunk_ nextLink URL (@odata.nextLink): %s" % nextLink) @@ -39,8 +38,9 @@ def handle_nextLink(helper=None, response=None, session=None): raise ValueError("nextLink scheme is not HTTPS. nextLink URL: %s" % nextLink) response = get_items_batch_session(helper=helper, url=nextLink, session=session) - items = response['value'] - return items + return response + else: + return None def requests_retry_session(retries=3, backoff_factor=1, status_forcelist=(429, 500, 502, 503, 504), session=None): session = session or requests.Session()