diff --git a/README.md b/README.md index 8b26dff..32e72da 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Turbo Data Miner -This extension adds a new tab `Turbo Miner` to Burp Suite's GUI as well as an new entry `Process in Turbo Data Miner +This extension adds a new tab `Turbo Miner` to Burp Suite's GUI as well as a new entry `Process in Turbo Data Miner (Proxy History Analyzer tab)` to Burp Suite's context menu. In the new tab, you are able to write new or select existing Python scripts that are executed on each request/response item currently stored in the Proxy History, Side Map, or on each request/response item that is sent or received by Burp Suite. @@ -12,7 +12,7 @@ the data collected and processed by Burp Suite. The following screenshot provides an example how Turbo Data Miner can be used to obtain a structured presentation of all cookies (and their attributes) that are stored in the current Burp Suite project. At the bottom (see 1), you select -the corresponding Python script in the combobox. By clicking button `Load Script`, the selected +the corresponding Python script in the combo box. By clicking button `Load Script`, the selected code is then loaded into the IDE text area and can be customized, if needed. Alternatively, you can create your own script by clicking button `New Script`. The analysis is started by clicking button `Start`. Afterwards, Turbo Data Miner executes the compiled Python script on each Request/Response item. Thereby, the script extracts cookie information @@ -22,10 +22,10 @@ column to gain a better understanding of each attribute or perform additional op ![Turbo Data Miner's Proxy History Analyzer](example.png) -As you can see, with Python skills, an understanding of the +As we can see, with Python skills, an understanding of the [Burp Suite Extender API](https://portswigger.net/Burp/extender/api/index.html) as well as an understanding of Turbo Miner's API (see Turbo Data Miner tab `About` or directly the -[HTML page](https://github.com/chopicalqui/TurboDataMiner/blob/master/turbominer/about.html) used by the `About` tab), +[HTML page](https://github.com/chopicalqui/TurboDataMiner/blob/master/turbodataminer/about.html) used by the `About` tab), you can extract and structure any information available in the current Burp Suite project. @@ -146,7 +146,8 @@ This tab contains the documentation about Turbo Intruder's Application Programmi # Author -**Lukas Reiter** (@chopicalquy) - [Turbo Data Miner](https://github.com/chopicalqui/TurboDataMiner) +**Lukas Reiter** ([@chopicalquy](https://twitter.com/chopicalquy)) - +[Turbo Data Miner](https://github.com/chopicalqui/TurboDataMiner) # License diff --git a/turbodataminer/about.html b/turbodataminer/about.html index 21e0925..29c6b0c 100755 --- a/turbodataminer/about.html +++ b/turbodataminer/about.html @@ -7,7 +7,7 @@

Turbo Data Miner

- This extension adds a new tab Turbo Miner to Burp Suite's GUI as well as an new entry + This extension adds a new tab Turbo Miner to Burp Suite's GUI as well as a new entry Process in Turbo Miner to Burp Suite's context menu. In the new tab, you are able to write new or select existing Python scripts that are executed on each request/response item currently stored in the Proxy History, Side Map, or on each request/response item that is sent or received by Burp Suite. @@ -762,15 +762,15 @@

Method Details

analyze_request(self, message_info)
This method returns an IRequestInfo object based on the given IHttpRequestResponse object.

- :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as a
+ :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as an
IRequestInfo object.
- :return (IRequestInfo): A IRequestInfo object or None, if no request was found.
+ :return (IRequestInfo): An IRequestInfo object or None, if no request was found.
analyze_response(self, message_info)
This method returns an IResponseInfo object based on the given IHttpRequestResponse object.

- :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as a
+ :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as an
IResponseInfo object.
- :return (IResponseInfo): A IResponseInfo object or None, if no response was found.
+ :return (IResponseInfo): An IResponseInfo object or None, if no response was found.
analyze_signatures(self, content, strict=False)
This method checks whether the given string matches one of the known file signatures specified in an internal
database.
diff --git a/turbodataminer/data/version-vulns.json b/turbodataminer/data/version-vulns.json index e1372fd..53ff34f 100644 --- a/turbodataminer/data/version-vulns.json +++ b/turbodataminer/data/version-vulns.json @@ -415,7 +415,7 @@ }, "Liferay, headers": { "alias": "cpe:/a:liferay:liferay_portal", - "regex": "Liferay-Portal:\\s*[a-z\\s]+([\\d.]+)", + "regex": "Liferay-Portal:\\s*[a-zA-Z\\s]+([\\d.]+)", "type": "cpe" }, "LinkSmart, script": { diff --git a/turbodataminer/scripts/15302930-aa43-4d7a-88c5-434bc4b9f763.json b/turbodataminer/scripts/15302930-aa43-4d7a-88c5-434bc4b9f763.json index 659f037..7574d29 100755 --- a/turbodataminer/scripts/15302930-aa43-4d7a-88c5-434bc4b9f763.json +++ b/turbodataminer/scripts/15302930-aa43-4d7a-88c5-434bc4b9f763.json @@ -6,6 +6,6 @@ ], "uuid": "15302930-aa43-4d7a-88c5-434bc4b9f763", "version": "v1.0", - "script": "\"\"\"\nThis script searches in-scope HTTP requests and responses for information based on regular expressions\nthat are stored in dictionary session[\"relist\"] and if found, adds the identified value to the table above.\n\nNote that each regular expression must contain the named group \"value\" whose content will be extracted\nadded to the table.\n\"\"\"\nimport re\n# Do the initial setup\nif ref == 1 or \"relist\" not in session:\n\tsession[\"relist\"] = {\n\t\t\t\t\t\t\t\t\t\t\t\"guid\": re.compile(\"[^0-9a-zA-Z](?P[0-9a-zA-Z]{8,8}(-[0-9a-zA-Z]{4,4}){3,3}-[0-9a-zA-Z]{12,12})[^0-9a-zA-Z]\"),\n\t\t\t\t\t\t\t\t\t\t\t#\"meta-tag\": re.compile(\"(?P)\")\n\t\t\t\t\t\t\t\t\t\t\t}\n\theader = [\"Ref.\", \"Host\", \"Extracted From\", \"Category\", \"Value\"]\n\ndef parse_content(content, regexes):\n\t\"\"\"\n\tThis method implements the core functionality to extract information\n\tfrom requests or responses based on the given regular expressions.\n\t\"\"\"\n\trvalues = {}\n\tfor key, value in regexes.items():\n\t\tfor match in value.finditer(content):\n\t\t\tif key in rvalues:\n\t\t\t\trvalues[key].append(match.group(\"value\"))\n\t\t\telse:\n\t\t\t\trvalues[key] = [match.group(\"value\")]\n\treturn rvalues\n\n# Process only in-scope requests and responses\nif in_scope:\n\trequest = message_info.getRequest()\n\tresponse = message_info.getResponse()\n\trequest_string = helpers.bytesToString(request).encode(\"utf-8\")\n\tresponse_string = helpers.bytesToString(response).encode(\"utf-8\") if response else \"\"\n\trequest_list = parse_content(request_string, session[\"relist\"])\n\tresponse_list = parse_content(response_string, session[\"relist\"])\n\tfor key, values in request_list.items():\n\t\trows.extend([[ref, url.getHost(), \"Request\", key, item] for item in values if item != url.getHost()])\n\tfor key, values in response_list.items():\n\t\trows.extend([[ref, url.getHost(), \"Response\", key, item] for item in values if item != url.getHost()])", + "script": "\"\"\"\nThis script searches in-scope HTTP requests and responses for information based on regular expressions\nthat are stored in dictionary session[\"relist\"] and if found, adds the identified value to the table above.\n\nNote that each regular expression must contain the named group \"value\" whose content will be extracted\nand added to the table.\n\"\"\"\nimport re\n# Do the initial setup\nif ref == 1 or \"relist\" not in session:\n\tsession[\"relist\"] = {\n\t\t\t\t\t\t\t\t\t\t\t\"guid\": re.compile(\"[^0-9a-zA-Z](?P[0-9a-zA-Z]{8,8}(-[0-9a-zA-Z]{4,4}){3,3}-[0-9a-zA-Z]{12,12})[^0-9a-zA-Z]\"),\n\t\t\t\t\t\t\t\t\t\t\t#\"meta-tag\": re.compile(\"(?P)\")\n\t\t\t\t\t\t\t\t\t\t\t}\n\theader = [\"Ref.\", \"Host\", \"Extracted From\", \"Category\", \"Value\"]\n\ndef parse_content(content, regexes):\n\t\"\"\"\n\tThis method implements the core functionality to extract information\n\tfrom requests or responses based on the given regular expressions.\n\t\"\"\"\n\trvalues = {}\n\tfor key, value in regexes.items():\n\t\tfor match in value.finditer(content):\n\t\t\tif key in rvalues:\n\t\t\t\trvalues[key].append(match.group(\"value\"))\n\t\t\telse:\n\t\t\t\trvalues[key] = [match.group(\"value\")]\n\treturn rvalues\n\n# Process only in-scope requests and responses\nif in_scope:\n\trequest = message_info.getRequest()\n\tresponse = message_info.getResponse()\n\trequest_string = helpers.bytesToString(request).encode(\"utf-8\")\n\tresponse_string = helpers.bytesToString(response).encode(\"utf-8\") if response else \"\"\n\trequest_list = parse_content(request_string, session[\"relist\"])\n\tresponse_list = parse_content(response_string, session[\"relist\"])\n\tfor key, values in request_list.items():\n\t\trows.extend([[ref, url.getHost(), \"Request\", key, item] for item in values if item != url.getHost()])\n\tfor key, values in response_list.items():\n\t\trows.extend([[ref, url.getHost(), \"Response\", key, item] for item in values if item != url.getHost()])", "name": "Misc - Template Script to Extract Information From In-Scope Requests and Responses" } \ No newline at end of file diff --git a/turbodataminer/scripts/45516c08-5289-4c66-ae30-8e7c69319e0a.json b/turbodataminer/scripts/45516c08-5289-4c66-ae30-8e7c69319e0a.json index 0f3fc69..f448908 100644 --- a/turbodataminer/scripts/45516c08-5289-4c66-ae30-8e7c69319e0a.json +++ b/turbodataminer/scripts/45516c08-5289-4c66-ae30-8e7c69319e0a.json @@ -6,6 +6,6 @@ ], "uuid": "45516c08-5289-4c66-ae30-8e7c69319e0a", "version": "v1.0", - "script": "\"\"\"\nThis script adds/from to relationships to the above table by analysing each message info item's Referer and Location header.\nThis information (especially first column of last row) can then be used to plot a relationship map using Python's matplotlib\nand networkx libraries by executing the following steps:\n1. Run the Turbo Data Miner Script\n\n2. Install necessary Python libraries\n$ pip install igviz matplotlib networkx numpy\n\n3. Setup Python environment\n>>> import matplotlib.pyplot as plt\n>>> import networkx as nx\n>>> import igviz as ig\n>>> DG = nx.DiGraph()\n\n4. Click into the first column of the last row, right click, and select 'Copy Cell Value'\n\n5. Select the below placeholder and paste the content of the clipboard\n>>> DG.add_weighted_edges_from([PLACEHOLDER])\n\n6a. Plot the directed graph using matplotlib\n>>> nx.draw(DG, with_labels=True, font_weight='bold')\n>>> plt.show()\n\n6b. Plot the directed graph using igviz\n>>> fig = ig.plot(DG)\n>>> fig.show()\n\"\"\"\n\nfrom java.net import URL\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"From\", \"To\"]\n\tsession[\"dedup\"] = {}\n\tsession[\"rows\"] = []\t\n\nrequest = message_info.getRequest()\nresponse = message_info.getResponse()\nif response:\n\thost_name = get_hostname(url)\n\trequest_info = helpers.analyzeRequest(request)\n\tresponse_info = helpers.analyzeResponse(response)\n\treferer = get_header(request_info.getHeaders(), \"Referer\")[1]\n\tlocation = get_header(response_info.getHeaders(), \"Location\")[1]\n\t# prepare referer and location\n\treferer_str = get_hostname(URL(referer)) if referer else None\n\tlocation_str = get_hostname(URL(location)) if location and location[0] != '/' else None\n\t# Add referer and location headers to table\n\tif referer_str and referer_str != host_name:\n\t\tkey = \"{}>{}\".format(referer_str, host_name)\n\t\tif key not in session[\"dedup\"]:\n\t\t\trows.append([referer_str, host_name])\n\t\t\tsession[\"rows\"].append((\"{}\".format(referer_str), \"{}\".format(host_name), 0.125))\n\t\t\tsession[\"dedup\"][key] = None\n\tif location_str and location_str != host_name:\n\t\tkey = \"{}>{}\".format(host_name, location_str)\n\t\tif key not in session[\"dedup\"]:\n\t\t\trows.append([host_name, location_str])\n\t\t\tsession[\"rows\"].append((\"{}\".format(host_name), \"{}\".format(location_str), 0.125))\n\t\t\tsession[\"dedup\"][key] = None\n# Add the list for the Python library networkx to the last row\nif ref == row_count:\n\trows.append([session[\"rows\"]])\n\t", + "script": "\"\"\"\nThis script adds/from to relationships to the above table by analysing each message info item's Referer and Location header.\nThis information (especially first column of last row) can then be used to plot a relationship map using Python's matplotlib\nand networkx libraries by executing the following steps:\n1. Run the Turbo Data Miner Script\n\n2. Install necessary Python libraries\n$ pip install igviz matplotlib networkx numpy\n\n3. Setup Python environment\n>>> import matplotlib.pyplot as plt\n>>> import networkx as nx\n>>> import igviz as ig\n>>> DG = nx.DiGraph()\n\n4. Click into the first column of the last row, right click, and select 'Copy Cell Value'\n\n5. Select the below PLACEHOLDER and replace it with the content of the clipboard\n>>> DG.add_weighted_edges_from([PLACEHOLDER])\n\n6a. Plot the directed graph using matplotlib\n>>> nx.draw(DG, with_labels=True, font_weight='bold')\n>>> plt.show()\n\n6b. Plot the directed graph using igviz\n>>> fig = ig.plot(DG)\n>>> fig.show()\n\"\"\"\nimport re\nfrom java.net import URL\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"From\", \"To\"]\n\tsession[\"dedup\"] = {}\n\tsession[\"rows\"] = []\n\nrequest = message_info.getRequest()\nresponse = message_info.getResponse()\nif response:\n\thost_name = get_hostname(url)\n\trequest_info = helpers.analyzeRequest(request)\n\tresponse_info = helpers.analyzeResponse(response)\n\treferer = get_header(request_info.getHeaders(), \"Referer\")[1]\n\tlocation = get_header(response_info.getHeaders(), \"Location\")[1]\n\t# prepare referer and location\n\treferer_str = get_hostname(URL(referer)) if referer else None\n\tlocation_str = get_hostname(URL(location)) if location and re.match(\"https?://\", location) else None\n\t# Add referer and location headers to table\n\tif referer_str and referer_str != host_name:\n\t\tkey = \"{}>{}\".format(referer_str, host_name)\n\t\tif key not in session[\"dedup\"]:\n\t\t\trows.append([referer_str, host_name])\n\t\t\tsession[\"rows\"].append((\"{}\".format(referer_str), \"{}\".format(host_name), 0.125))\n\t\t\tsession[\"dedup\"][key] = None\n\tif location_str and location_str != host_name:\n\t\tkey = \"{}>{}\".format(host_name, location_str)\n\t\tif key not in session[\"dedup\"]:\n\t\t\trows.append([host_name, location_str])\n\t\t\tsession[\"rows\"].append((\"{}\".format(host_name), \"{}\".format(location_str), 0.125))\n\t\t\tsession[\"dedup\"][key] = None\n# Add the list for the Python library networkx to the last row\nif ref == row_count:\n\trows.append([session[\"rows\"]])\n\t", "name": "Misc - Template Script to Visualize Web Site Relationships" } \ No newline at end of file diff --git a/turbodataminer/scripts/4a70691d-14fd-4fea-815c-9eef43c560a9.json b/turbodataminer/scripts/4a70691d-14fd-4fea-815c-9eef43c560a9.json index 99d3af6..09e5ff3 100644 --- a/turbodataminer/scripts/4a70691d-14fd-4fea-815c-9eef43c560a9.json +++ b/turbodataminer/scripts/4a70691d-14fd-4fea-815c-9eef43c560a9.json @@ -6,6 +6,6 @@ ], "uuid": "4a70691d-14fd-4fea-815c-9eef43c560a9", "version": "v1.0", - "script": "\"\"\"\nThis script parses XML files and displays each leaf tag together with its attributes and values in the table above.\n\nThis script parses the HTTP response body for XML objects and displays each leaf tag together with its\nattributes and values in the table above.\nUse this script to identify the location of a specific value within the XML object or to reduce the complexity of\nthe XML object during a review.\n\"\"\"\nimport os\nimport re\nfrom java.lang import Thread\n\n# Due to the following issue, we have to manually load our own local Apache Xerces library:\n# https://forum.portswigger.net/thread/saxparser-dependency-delimma-499c057a\nThread.currentThread().setContextClassLoader(xerceslib)\nimport xml.etree.ElementTree as ET\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Type\", \"Name\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(tag, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given XML tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\ttag_name = re.sub(\"^\\{http://.*?\\}\", \"\", tag.tag)\n\tresult = []\n\tnew_path = os.path.join(path, tag_name)\n\tif len(list(tag)) == 0:\n\t\tresult.append([ref, url, new_path, \"Tag\", unicode(tag_name), unicode(tag.text)])\n\t\tfor attribute in tag.items():\n\t\t\tresult.append([ref, url, \"{}/@{}\".format(new_path, attribute[0]), \"Attribute\", unicode(attribute[0]), unicode(attribute[1])])\n\telse:\n\t\tfor item in list(tag):\n\t\t\tresult += get_items(item, url, ref, new_path)\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\troot = ET.fromstring(body_content.encode(\"utf-8\"))\n\t\tresults = get_items(root, url, ref)\n \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\tpass", + "script": "\"\"\"\nThis script parses the HTTP response body for XML objects and displays each leaf tag together with its\nattributes and values in the table above.\n\nUse this script to identify the location of a specific value within the XML object or to reduce the complexity of\nthe XML object during a review.\n\"\"\"\nimport os\nimport re\nimport traceback\nfrom java.lang import Thread\n\n# Due to the following issue, we have to manually load our own local Apache Xerces library:\n# https://forum.portswigger.net/thread/saxparser-dependency-delimma-499c057a\nThread.currentThread().setContextClassLoader(xerceslib)\nimport xml.etree.ElementTree as ET\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Type\", \"Name\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(tag, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given XML tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\ttag_name = re.sub(\"^\\{http://.*?\\}\", \"\", tag.tag)\n\tresult = []\n\tnew_path = os.path.join(path, tag_name)\n\tif len(list(tag)) == 0:\n\t\tresult.append([ref, url, new_path, \"Tag\", unicode(tag_name), unicode(tag.text)])\n\t\tfor attribute in tag.items():\n\t\t\tresult.append([ref, url, \"{}/@{}\".format(new_path, attribute[0]), \"Attribute\", unicode(attribute[0]), unicode(attribute[1])])\n\telse:\n\t\tfor item in list(tag):\n\t\t\tresult += get_items(item, url, ref, new_path)\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\troot = ET.fromstring(body_content.encode(\"utf-8\"))\n\t\tresults = get_items(root, url, ref)\n \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\ttraceback.print_exc(file=callbacks.getStderr())", "name": "XML - Template Script to Display All XML Leaf Tag and Attribute Values (Deduplicated) From Responses" } \ No newline at end of file diff --git a/turbodataminer/scripts/aa0b7080-f27c-4ed4-a6a2-7802c6656a64.json b/turbodataminer/scripts/aa0b7080-f27c-4ed4-a6a2-7802c6656a64.json index 349b6ba..399fb11 100755 --- a/turbodataminer/scripts/aa0b7080-f27c-4ed4-a6a2-7802c6656a64.json +++ b/turbodataminer/scripts/aa0b7080-f27c-4ed4-a6a2-7802c6656a64.json @@ -6,6 +6,6 @@ ], "uuid": "aa0b7080-f27c-4ed4-a6a2-7802c6656a64", "version": "v1.0", - "script": "\"\"\"\nThis script tries to convert HTTP response bodies into JSON objects and afterwards tries to extract the \nvalues of the JSON attributes that are specified in variable session[\"attributes\"]. Note that the specified \nJSON attributes have to exist within the same hierarchy level in order to be added to the table above.\n\nIn addition, variable session[\"matches\"] specifies how many of the specified attributes must exist within \nthe same hierarchy level in order to be added to the table.\n\"\"\"\n# Do the initial setup\n# Update the contents of session[\"attributes\"] and session[\"matches\"] accordingly\nif ref == 1 or \"attributes\" not in session or \"matches\" not in session:\n\tsession[\"attributes\"] = [\"asset_identifier\", \"asset_type\"]\n\tsession[\"matches\"] = 2 #len(session[\"attributes\"])\n\theader = [\"ref\", \"url\"]\n\theader.extend(session[\"attributes\"])\n\n# Process only in-scope HTTP responses\nresponse = message_info.getResponse()\nif in_scope and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_string = helpers.bytesToString(body_bytes)\n\ttry:\n\t\trvalues = get_json_attributes(body_string, session[\"attributes\"], session[\"matches\"])\n\t\tfor item in rvalues:\n\t\t\tvalues = [ref, url]\n\t\t\tfor key in session[\"attributes\"]:\n\t\t\t\tvalues.append(item[key])\n\t\t\trows.append(values)\n\texcept Exception:\n\t\tpass", + "script": "\"\"\"\nThis script tries to convert HTTP response bodies into JSON objects and afterwards tries to extract the \nvalues of the JSON attributes that are specified in variable session[\"attributes\"]. Note that the specified \nJSON attributes have to exist within the same hierarchy level in order to be added to the table above.\n\nIn addition, variable session[\"matches\"] specifies how many of the specified attributes must exist within \nthe same hierarchy level in order to be added to the table.\n\"\"\"\nimport traceback\n# Do the initial setup\n# Update the contents of session[\"attributes\"] and session[\"matches\"] accordingly\nif ref == 1 or \"attributes\" not in session or \"matches\" not in session:\n\tsession[\"attributes\"] = [\"asset_identifier\", \"asset_type\"]\n\tsession[\"matches\"] = 2 #len(session[\"attributes\"])\n\theader = [\"ref\", \"url\"]\n\theader.extend(session[\"attributes\"])\n\n# Process only in-scope HTTP responses\nresponse = message_info.getResponse()\nif in_scope and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_string = helpers.bytesToString(body_bytes)\n\ttry:\n\t\trvalues = get_json_attributes(body_string, session[\"attributes\"], session[\"matches\"])\n\t\tfor item in rvalues:\n\t\t\tvalues = [ref, url]\n\t\t\tfor key in session[\"attributes\"]:\n\t\t\t\tvalues.append(item[key])\n\t\t\trows.append(values)\n\texcept:\n\t\ttraceback.print_exc(file=callbacks.getStderr())", "name": "JSON - Template Script to Extract JSON Attribute Values From Responses" } \ No newline at end of file diff --git a/turbodataminer/scripts/d8dbb2e9-1765-4319-baef-81ba36d3654b.json b/turbodataminer/scripts/d8dbb2e9-1765-4319-baef-81ba36d3654b.json index a269c4b..ae7ba3c 100644 --- a/turbodataminer/scripts/d8dbb2e9-1765-4319-baef-81ba36d3654b.json +++ b/turbodataminer/scripts/d8dbb2e9-1765-4319-baef-81ba36d3654b.json @@ -6,6 +6,6 @@ ], "uuid": "d8dbb2e9-1765-4319-baef-81ba36d3654b", "version": "v1.0", - "script": "\"\"\"\nThis script parses the HTTP response body for JSON objects and displays each leaf attribute together with its\nvalues in the table above.\nUse this script to identify the location of a specific value within the JSON object or to reduce the complexity of\nthe JSON object during a review.\n\"\"\"\nimport os\nimport json\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(content, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given JSON object tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\tresult = []\n\tif isinstance(content, dict):\n\t\tfor key, value in content.items():\n\t\t\tresult += get_items(value, url, ref, os.path.join(path, str(key)))\n\telif isinstance(content, list):\n\t\tfor item in content:\n\t\t\tresult += get_items(item, url, ref, path)\n\telse:\n\t\tresult = [[ref, url, unicode(path), unicode(content)]]\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\tjson_object = json.JSONDecoder().decode(body_content)\n\t\tresults = get_items(json_object, url, ref)\n \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\tpass", + "script": "\"\"\"\nThis script parses the HTTP response body for JSON objects and displays each leaf attribute together with its\nvalues in the table above.\n\nUse this script to identify the location of a specific value within the JSON object or to reduce the complexity of\nthe JSON object during a review.\n\"\"\"\nimport os\nimport json\nimport traceback\n\nif ref == 1 or \"dedup\" not in session:\n\theader = [\"Ref.\", \"URL\", \"Path\", \"Value\"]\n\t# If you want to disable deduplication, remove the following line and press button \"Clear Session\" to \n\t# reset the content of the session variable\n\tsession[\"dedup\"] = {}\n\ndef get_items(content, url, ref, path=\"/\"):\n\t\"\"\"\n\tThis method recursively parses the given JSON object tag and returns the results in a two-dimensional list.\n\t\"\"\"\n\tresult = []\n\tif isinstance(content, dict):\n\t\tfor key, value in content.items():\n\t\t\tresult += get_items(value, url, ref, os.path.join(path, unicode(key)))\n\telif isinstance(content, list):\n\t\tfor item in content:\n\t\t\tresult += get_items(item, url, ref, path)\n\telse:\n\t\tresult = [[ref, url, unicode(path), unicode(content)]]\n\treturn result\n\nresponse = message_info.getResponse()\nif True and response:\n\tresponse_info = helpers.analyzeResponse(response)\n\tbody_offset = response_info.getBodyOffset()\n\tbody_bytes = response[body_offset:]\n\tbody_content = helpers.bytesToString(body_bytes)\n\t\n\ttry:\n\t\tjson_object = json.JSONDecoder().decode(body_content)\n\t\tresults = get_items(json_object, url, ref)\n \t# perform deduplication\n\t\tif \"dedup\" in session:\n\t\t\tfor row in results:\n\t\t\t\tkey = \":\".join([unicode(item) for item in row[1:]])\n\t\t\t\tif key not in session[\"dedup\"]:\n\t\t\t\t\trows.append(row)\n\t\t\t\t\tsession[\"dedup\"][key] = None\n\t\telse:\n\t\t\trows = results\n\texcept:\n\t\ttraceback.print_exc(file=callbacks.getStderr())", "name": "JSON - Template Script to Display All Leaf JSON Attribute Values (Deduplicated) From Responses" } \ No newline at end of file diff --git a/turbodataminer/turbodataminer.py b/turbodataminer/turbodataminer.py index 900f21e..6f45ef2 100755 --- a/turbodataminer/turbodataminer.py +++ b/turbodataminer/turbodataminer.py @@ -1160,9 +1160,9 @@ def analyze_request(self, message_info): """ This method returns an IRequestInfo object based on the given IHttpRequestResponse object. - :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as a + :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as an IRequestInfo object. - :return (IRequestInfo): A IRequestInfo object or None, if no request was found. + :return (IRequestInfo): An IRequestInfo object or None, if no request was found. """ request = message_info.getRequest() if request: @@ -1175,9 +1175,9 @@ def analyze_response(self, message_info): """ This method returns an IResponseInfo object based on the given IHttpRequestResponse object. - :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as a + :param message_info (IHttpRequestResponse): The IHttpRequestResponse whose request should be returned as an IResponseInfo object. - :return (IResponseInfo): A IResponseInfo object or None, if no response was found. + :return (IResponseInfo): An IResponseInfo object or None, if no response was found. """ response = message_info.getResponse() if response: @@ -1559,6 +1559,8 @@ def get_json_attribute_by_path(self, body, path, default_value=None): path = path[1:] if path[0] == '/' else path current_position = body if isinstance(body, dict) else json.JSONDecoder().decode(body) for value in path.split("/"): + if not self._ide_pane.activated: + return current_position if isinstance(current_position, dict) and value in current_position: current_position = current_position[value] else: