Add new script for JSON object analyses

README.md

@@ -1,9 +1,9 @@
 # Turbo Data Miner
 
-This extension adds a new tab `Turbo Miner` to Burp Suite's GUI as well as an new entry `Process in Turbo Miner` to 
-Burp Suite's context menu. In the new tab, you are able to write new or select existing Python scripts that are 
-executed on each request/response item currently stored in the Proxy History, Side Map, or on each request/response 
-item that is sent or received by Burp Suite.  
+This extension adds a new tab `Turbo Miner` to Burp Suite's GUI as well as an new entry `Process in Turbo Data Miner 
+(Proxy History Analyzer tab)` to Burp Suite's context menu. In the new tab, you are able to write new or select 
+existing Python scripts that are executed on each request/response item currently stored in the Proxy History, Side 
+Map, or on each request/response item that is sent or received by Burp Suite.  
 
 The objective of these Python scripts is the flexible and dynamic extraction, correlation, and structured
 presentation of information from the Burp Suite state as well as the flexible and dynamic on-the-fly modification

turbodataminer/scripts/4a70691d-14fd-4fea-815c-9eef43c560a9.json

@@ -6,6 +6,6 @@
     ], 
     "uuid": "4a70691d-14fd-4fea-815c-9eef43c560a9", 
     "version": "v1.0", 
-    "script": "\"\"\"\nThis script parses XML files and displays each leaf tag together with its attributes and values in the table above.\n\"\"\"\nimport os\nimport re\nfrom java.lang import Thread\n\n# Due to the following issue, we have to manually load our own local Apache Xerces library:\n# https://forum.portswigger.net/thread/saxparser-dependency-delimma-499c057a\nThread.currentThread().setContextClassLoader(xerceslib)\nimport xml.etree.ElementTree as ET\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Type\", \"Name\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(tag, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given XML tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\ttag_name = re.sub(\"^\\{http://.*?\\}\", \"\", tag.tag)\n\tresult = []\n\tnew_path = os.path.join(path, tag_name)\n\tif len(list(tag)) == 0:\n\t\tresult.append([ref, url, new_path, \"Tag\", tag_name, tag.text])\n\t\tfor attribute in tag.items():\n\t\t\tresult.append([ref, url, \"{}/@{}\".format(new_path, attribute[0]), \"Attribute\",  attribute[0], attribute[1]])\n\telse:\n\t\tfor item in list(tag):\n\t\t\tresult += get_items(item, url, ref, new_path)\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\troot = ET.fromstring(body_content.encode(\"utf-8\"))\n\t\tresults = get_items(root, url, ref)\n    \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\tpass", 
-    "name": "XML - Template Script to Extract XML Tag and Attribute Values From Responses"
+    "script": "\"\"\"\nThis script parses XML files and displays each leaf tag together with its attributes and values in the table above.\n\nThis script parses the HTTP response body for XML objects and displays each leaf tag together with its\nattributes and values in the table above.\nUse this script to identify the location of a specific value within the XML object or to reduce the complexity of\nthe XML object during a review.\n\"\"\"\nimport os\nimport re\nfrom java.lang import Thread\n\n# Due to the following issue, we have to manually load our own local Apache Xerces library:\n# https://forum.portswigger.net/thread/saxparser-dependency-delimma-499c057a\nThread.currentThread().setContextClassLoader(xerceslib)\nimport xml.etree.ElementTree as ET\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Type\", \"Name\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(tag, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given XML tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\ttag_name = re.sub(\"^\\{http://.*?\\}\", \"\", tag.tag)\n\tresult = []\n\tnew_path = os.path.join(path, tag_name)\n\tif len(list(tag)) == 0:\n\t\tresult.append([ref, url, new_path, \"Tag\", unicode(tag_name), unicode(tag.text)])\n\t\tfor attribute in tag.items():\n\t\t\tresult.append([ref, url, \"{}/@{}\".format(new_path, attribute[0]), \"Attribute\",  unicode(attribute[0]), unicode(attribute[1])])\n\telse:\n\t\tfor item in list(tag):\n\t\t\tresult += get_items(item, url, ref, new_path)\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\troot = ET.fromstring(body_content.encode(\"utf-8\"))\n\t\tresults = get_items(root, url, ref)\n    \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\tpass", 
+    "name": "XML - Template Script to Display All XML Leaf Tag and Attribute Values (Deduplicated) From Responses"
 }

turbodataminer/scripts/d8dbb2e9-1765-4319-baef-81ba36d3654b.json

@@ -0,0 +1,11 @@
+{
+    "author": "Lukas Reiter", 
+    "plugins": [
+        0, 
+        6
+    ], 
+    "uuid": "d8dbb2e9-1765-4319-baef-81ba36d3654b", 
+    "version": "v1.0", 
+    "script": "\"\"\"\nThis script parses the HTTP response body for JSON objects and displays each leaf attribute together with its\nvalues in the table above.\nUse this script to identify the location of a specific value within the JSON object or to reduce the complexity of\nthe JSON object during a review.\n\"\"\"\nimport os\nimport json\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(content, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given JSON object tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\tresult = []\n\tif isinstance(content, dict):\n\t\tfor key, value in content.items():\n\t\t\tresult += get_items(value, url, ref, os.path.join(path, str(key)))\n\telif isinstance(content, list):\n\t\tfor item in content:\n\t\t\tresult += get_items(item, url, ref, path)\n\telse:\n\t\tresult = [[ref, url, unicode(path), unicode(content)]]\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\tjson_object = json.JSONDecoder().decode(body_content)\n\t\tresults = get_items(json_object, url, ref)\n    \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\tpass", 
+    "name": "JSON - Template Script to Display All Leaf JSON Attribute Values (Deduplicated) From Responses"
+}