From 33b9ed1f8bf3b6382fcf4dafb2268b947edc311a Mon Sep 17 00:00:00 2001 From: Matthias Duve Date: Mon, 8 Jan 2024 16:21:16 +0100 Subject: [PATCH 01/19] Added Humble --- artemis/modules/humble.py | 132 ++++++++++++++++++ artemis/reporting/modules/humble/reporter.py | 68 +++++++++ .../template_missing_security_header.jinja2 | 20 +++ docker-compose.yaml | 10 ++ docker/Dockerfile | 3 + test/modules/test_humble.py | 47 +++++++ 6 files changed, 280 insertions(+) create mode 100644 artemis/modules/humble.py create mode 100644 artemis/reporting/modules/humble/reporter.py create mode 100644 artemis/reporting/modules/humble/template_missing_security_header.jinja2 create mode 100644 test/modules/test_humble.py diff --git a/artemis/modules/humble.py b/artemis/modules/humble.py new file mode 100644 index 000000000..326e039a5 --- /dev/null +++ b/artemis/modules/humble.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +import json +import subprocess +from typing import Any, Dict, List + +from karton.core import Task + +from artemis.binds import Service, TaskStatus, TaskType +from artemis.module_base import ArtemisBase +from artemis.task_utils import get_target_url + + +def process_json_data(result: Dict[str, Any]) -> List[str]: + messages = [] + + # Iterate through key-value pairs in the result + for key, value in result.items(): + # Split the key to extract the relevant part + key_parts = key.replace("[", "").replace("]", "").split(". ") + + # Check if the key has the expected structure + if len(key_parts) >= 2: + category = key_parts[1].capitalize() + + # Check if the key is not in the excluded categories and there are relevant values + if ( + category.lower() != "info" + and key + not in [ + "[2. Fingerprint HTTP Response Headers]", + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]", + "[4. Empty HTTP Response Headers Values]", + "[5. Browser Compatibility for Enabled HTTP Security Headers]", + ] + and ( + isinstance(value, dict) + or ( + isinstance(value, list) + and any( + subvalue + for subvalue in value + if subvalue + and subvalue + not in ["Nothing to report, all seems OK!", "No HTTP security headers are enabled."] + ) + ) + ) + ): + messages.append(f"{category}:") + + # If the value is a dictionary, iterate through subkey-value pairs + if isinstance(value, dict): + for subkey, subvalue in value.items(): + # Add subkeys and subvalues to messages + if subvalue and subvalue not in [ + "Nothing to report, all seems OK!", + "No HTTP security headers are enabled.", + ]: + messages.append(f" {subkey}: {subvalue}") + + # If the value is a list, iterate through list items + elif isinstance(value, list): + for item in value: + # Add list items to messages + if item and item not in [ + "Nothing to report, all seems OK!", + "No HTTP security headers are enabled.", + ]: + messages.append(f" {item}") + + return messages + + +class Humble(ArtemisBase): + """ + Runs humble -> A HTTP Headers Analyzer + """ + + identity = "humble" + filters = [ + {"type": TaskType.SERVICE.value, "service": Service.HTTP.value}, + ] + + def run(self, current_task: Task) -> None: + base_url = get_target_url(current_task) + + data = subprocess.check_output( + [ + "python3", + "humble.py", + "-u", + base_url, + "-b", + "-o", + "json", + ], + cwd="/humble", + stderr=subprocess.DEVNULL, + ) + + # strip boilerplatetext from the output to get the location and filename of the output file + filename = ( + data.decode("ascii", errors="ignore") + .removeprefix("\n Analyzing URL and saving the report, please wait ...\n\n\n Report saved to ") + .removesuffix("\n") + ) + data_str = open(filename, "r").read() + + # cleanup file + subprocess.run(["rm", filename]) + + # Check if the input string is empty + if data_str.strip(): + result = json.loads(data_str) + else: + result = [] + + # Parse the JSON data + messages = process_json_data(result) + + if messages: + status = TaskStatus.INTERESTING + status_reason = messages[0] + ", ".join(messages[1:]) + else: + status = TaskStatus.OK + status_reason = None + + self.db.save_task_result(task=current_task, status=status, status_reason=status_reason, data=result) + + +if __name__ == "__main__": + Humble().loop() diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py new file mode 100644 index 000000000..c9414f50d --- /dev/null +++ b/artemis/reporting/modules/humble/reporter.py @@ -0,0 +1,68 @@ +from pathlib import Path +from typing import Any, Callable, Dict, List + +from artemis.reporting.base.language import Language +from artemis.reporting.base.normal_form import ( + NormalForm, + get_domain_normal_form, + get_domain_score, +) +from artemis.reporting.base.report import Report +from artemis.reporting.base.report_type import ReportType +from artemis.reporting.base.reporter import Reporter +from artemis.reporting.base.templating import ReportEmailTemplateFragment +from artemis.reporting.utils import get_top_level_target + + +class HumbleReporter(Reporter): + MISSING_SECURITY_HEADER = ReportType("missing_security_header") + + @staticmethod + def create_reports(task_result: Dict[str, Any], language: Language) -> List[Report]: + if task_result["headers"]["receiver"] != "humble": + return [] + + if not isinstance(task_result["result"], list): + return [] + + result = [] + for item in task_result["result"]: + if item["confidence"] == "CONFIRMED": + result.append( + Report( + top_level_target=get_top_level_target(task_result), + target=item["domain"], + report_type=HumbleReporter.MISSING_SECURITY_HEADER, + additional_data={ + "message_en": item["info"], + }, + timestamp=task_result["created_at"], + ) + ) + return result + + @staticmethod + def get_email_template_fragments() -> List[ReportEmailTemplateFragment]: + return [ + ReportEmailTemplateFragment.from_file( + str(Path(__file__).parents[0] / "template_missing_security_header.jinja2"), priority=2 + ), + ] + + @staticmethod + def get_scoring_rules() -> Dict[ReportType, Callable[[Report], List[int]]]: + """See the docstring in the parent class.""" + return {HumbleReporter.MISSING_SECURITY_HEADER: lambda report: [get_domain_score(report.target)]} + + @staticmethod + def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]: + """See the docstring in the Reporter class.""" + return { + HumbleReporter.MISSING_SECURITY_HEADER: lambda report: Reporter.dict_to_tuple( + { + "type": report.report_type, + "target": get_domain_normal_form(report.target), + "message": report.additional_data["message_en"], + } + ) + } diff --git a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 new file mode 100644 index 000000000..58cf1aa6b --- /dev/null +++ b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 @@ -0,0 +1,20 @@ +{% if "missing_security_header" in data.contains_type %} +
  • {% trans %}We identified that not all possible security headers are set correctly:{% endtrans %} + +

    + {% trans trimmed %} + Please verify the configuration, and, if a security header is missing, change it. Security + headers can have a deep impact in protecting your application against attacks. + {% endtrans %} +

    +
    • +{% endif %} diff --git a/docker-compose.yaml b/docker-compose.yaml index da84a9469..c7310c0c2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -357,6 +357,16 @@ services: restart: always volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] + karton-humble: + build: + context: . + dockerfile: docker/Dockerfile + volumes: [ "./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/" ] + depends_on: [ karton-logger, redis ] + env_file: .env + restart: always + command: "python3 -m artemis.modules.humble" + volumes: data-mongodb: data-redis: diff --git a/docker/Dockerfile b/docker/Dockerfile index b0b177879..0af0cbfee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,9 @@ RUN git clone https://github.com/projectdiscovery/nuclei.git -b v2.9.7 /nuclei COPY docker/patches/nuclei-rate-limiting.patch /nuclei/ RUN cd /nuclei && git apply nuclei-rate-limiting.patch && cd v2/cmd/nuclei && go build && GOBIN=/usr/local/bin/ go install +RUN git clone https://github.com/rfc-st/humble.git --branch master /humble +RUN pip install -r /humble/requirements.txt + COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt $ADDITIONAL_REQUIREMENTS diff --git a/test/modules/test_humble.py b/test/modules/test_humble.py new file mode 100644 index 000000000..16835719d --- /dev/null +++ b/test/modules/test_humble.py @@ -0,0 +1,47 @@ +import unittest +from unittest.mock import patch, mock_open, call +from karton.core import Task +import subprocess +from artemis.modules.humble import Humble +from artemis.binds import TaskStatus + +class TestHumble(unittest.TestCase): + def setUp(self): + self.humble = Humble() + + @patch('subprocess.check_output') + @patch('subprocess.run') + @patch('builtins.open', new_callable=mock_open, read_data='{"key": "value"}') + def test_run_with_messages(self, mock_file, mock_run, mock_output): + mock_output.return_value = b'filename' + task = Task({"type": "DOMAIN"}, payload={"domain": "test.com"}) + self.humble.run(task) + mock_output.assert_called_with( + ["python3", "humble.py", "-u", "https://test.com", "-b", "-o", "json"], + cwd="/humble", + stderr=subprocess.DEVNULL, + ) + mock_run.assert_called_with(["rm", "filename"]) + self.humble.db.save_task_result.assert_called_with( + task=task, status=TaskStatus.INTERESTING, status_reason='key: value', data={'key': 'value'} + ) + + @patch('subprocess.check_output') + @patch('subprocess.run') + @patch('builtins.open', new_callable=mock_open, read_data='') + def test_run_without_messages(self, mock_file, mock_run, mock_output): + mock_output.return_value = b'filename' + task = Task({"type": "DOMAIN"}, payload={"domain": "test.com"}) + self.humble.run(task) + mock_output.assert_called_with( + ["python3", "humble.py", "-u", "https://test.com", "-b", "-o", "json"], + cwd="/humble", + stderr=subprocess.DEVNULL, + ) + mock_run.assert_called_with(["rm", "filename"]) + self.humble.db.save_task_result.assert_called_with( + task=task, status=TaskStatus.OK, status_reason=None, data=[] + ) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From d8d666a094d228e6efaad24107f4a6ec9ddb061b Mon Sep 17 00:00:00 2001 From: Matthias Duve Date: Mon, 8 Jan 2024 16:40:51 +0100 Subject: [PATCH 02/19] Removed not functional test --- test/modules/test_humble.py | 78 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/test/modules/test_humble.py b/test/modules/test_humble.py index 16835719d..f9416bb36 100644 --- a/test/modules/test_humble.py +++ b/test/modules/test_humble.py @@ -1,47 +1,41 @@ import unittest -from unittest.mock import patch, mock_open, call -from karton.core import Task -import subprocess -from artemis.modules.humble import Humble -from artemis.binds import TaskStatus -class TestHumble(unittest.TestCase): - def setUp(self): - self.humble = Humble() +from artemis.modules import humble - @patch('subprocess.check_output') - @patch('subprocess.run') - @patch('builtins.open', new_callable=mock_open, read_data='{"key": "value"}') - def test_run_with_messages(self, mock_file, mock_run, mock_output): - mock_output.return_value = b'filename' - task = Task({"type": "DOMAIN"}, payload={"domain": "test.com"}) - self.humble.run(task) - mock_output.assert_called_with( - ["python3", "humble.py", "-u", "https://test.com", "-b", "-o", "json"], - cwd="/humble", - stderr=subprocess.DEVNULL, - ) - mock_run.assert_called_with(["rm", "filename"]) - self.humble.db.save_task_result.assert_called_with( - task=task, status=TaskStatus.INTERESTING, status_reason='key: value', data={'key': 'value'} - ) - @patch('subprocess.check_output') - @patch('subprocess.run') - @patch('builtins.open', new_callable=mock_open, read_data='') - def test_run_without_messages(self, mock_file, mock_run, mock_output): - mock_output.return_value = b'filename' - task = Task({"type": "DOMAIN"}, payload={"domain": "test.com"}) - self.humble.run(task) - mock_output.assert_called_with( - ["python3", "humble.py", "-u", "https://test.com", "-b", "-o", "json"], - cwd="/humble", - stderr=subprocess.DEVNULL, - ) - mock_run.assert_called_with(["rm", "filename"]) - self.humble.db.save_task_result.assert_called_with( - task=task, status=TaskStatus.OK, status_reason=None, data=[] - ) +class TestProcessJsonData(unittest.TestCase): + def test_process_json_data_with_valid_input(self) -> None: + # Setup + input_data = { + "[0. Info]": { + "Date": "1970/01/01 - 12:12:12", + "URL": "https://test.tld" + }, + "[1. Missing HTTP Security Headers]": [ + "Cache-Control", + "Clear-Site-Data", + "Cross-Origin-Embedder-Policy" + ], + "[2. Fingerprint HTTP Response Headers]": [ + "Server", + ], + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ + "X-XSS-Protection (Unsafe Value)" + ], + "[4. Empty HTTP Response Headers Values]": [ + "Content-Security-Policy" + ], + "[5. Browser Compatibility for Enabled HTTP Security Headers]": { + "X-XSS-Protection": "https://caniuse.com/?search=X-XSS-Protection" + } + } -if __name__ == '__main__': - unittest.main() \ No newline at end of file + # Exercise + result = humble.process_json_data(input_data) + + # Verify + expected_result = ["Missing http security headers:", "Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"] + self.assertEqual(result, expected_result) + +if __name__ == "__main__": + unittest.main() From 369fefc5a86f5c1fc7550e4dbc4886bb9c960615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:40:26 +0100 Subject: [PATCH 03/19] Minor improvements --- artemis/modules/humble.py | 41 +++++++++++++---- artemis/reporting/modules/humble/reporter.py | 46 ++++++++----------- .../template_missing_security_header.jinja2 | 16 ++++--- artemis/reporting/severity.py | 1 + test/modules/test_humble.py | 29 +++++------- 5 files changed, 74 insertions(+), 59 deletions(-) diff --git a/artemis/modules/humble.py b/artemis/modules/humble.py index 326e039a5..ed42231be 100644 --- a/artemis/modules/humble.py +++ b/artemis/modules/humble.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import dataclasses import json import subprocess from typing import Any, Dict, List @@ -10,8 +11,18 @@ from artemis.task_utils import get_target_url -def process_json_data(result: Dict[str, Any]) -> List[str]: - messages = [] +@dataclasses.dataclass +class Message: + category: str + problems: List[str] + + @property + def message(self) -> str: + return f"{self.category}: {', '.join(self.problems)}" + + +def process_json_data(result: Dict[str, Any]) -> List[Message]: + messages: Dict[str, Message] = {} # Iterate through key-value pairs in the result for key, value in result.items(): @@ -46,8 +57,6 @@ def process_json_data(result: Dict[str, Any]) -> List[str]: ) ) ): - messages.append(f"{category}:") - # If the value is a dictionary, iterate through subkey-value pairs if isinstance(value, dict): for subkey, subvalue in value.items(): @@ -56,7 +65,10 @@ def process_json_data(result: Dict[str, Any]) -> List[str]: "Nothing to report, all seems OK!", "No HTTP security headers are enabled.", ]: - messages.append(f" {subkey}: {subvalue}") + problem = f"{subkey} {subvalue}" + if category not in messages: + messages[category] = Message(category=category, problems=[]) + messages[category].problems.append(problem) # If the value is a list, iterate through list items elif isinstance(value, list): @@ -66,9 +78,11 @@ def process_json_data(result: Dict[str, Any]) -> List[str]: "Nothing to report, all seems OK!", "No HTTP security headers are enabled.", ]: - messages.append(f" {item}") + if category not in messages: + messages[category] = Message(category=category, problems=[]) + messages[category].problems.append(item) - return messages + return list(messages.values()) class Humble(ArtemisBase): @@ -120,12 +134,21 @@ def run(self, current_task: Task) -> None: if messages: status = TaskStatus.INTERESTING - status_reason = messages[0] + ", ".join(messages[1:]) + status_reason = ", ".join([message.message for message in messages]) else: status = TaskStatus.OK status_reason = None - self.db.save_task_result(task=current_task, status=status, status_reason=status_reason, data=result) + self.db.save_task_result( + task=current_task, + status=status, + status_reason=status_reason, + data={ + "original_result": result, + "message_data": [dataclasses.asdict(message) for message in messages], + "messages": [message.message for message in messages], + }, + ) if __name__ == "__main__": diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py index c9414f50d..b169ab1c6 100644 --- a/artemis/reporting/modules/humble/reporter.py +++ b/artemis/reporting/modules/humble/reporter.py @@ -4,41 +4,40 @@ from artemis.reporting.base.language import Language from artemis.reporting.base.normal_form import ( NormalForm, - get_domain_normal_form, - get_domain_score, + get_url_normal_form, + get_url_score, ) from artemis.reporting.base.report import Report from artemis.reporting.base.report_type import ReportType from artemis.reporting.base.reporter import Reporter from artemis.reporting.base.templating import ReportEmailTemplateFragment -from artemis.reporting.utils import get_top_level_target +from artemis.reporting.utils import get_target_url, get_top_level_target class HumbleReporter(Reporter): - MISSING_SECURITY_HEADER = ReportType("missing_security_header") + MISSING_SECURITY_HEADERS = ReportType("missing_security_headers") @staticmethod def create_reports(task_result: Dict[str, Any], language: Language) -> List[Report]: if task_result["headers"]["receiver"] != "humble": return [] - if not isinstance(task_result["result"], list): + if not isinstance(task_result["result"], dict): return [] result = [] - for item in task_result["result"]: - if item["confidence"] == "CONFIRMED": - result.append( - Report( - top_level_target=get_top_level_target(task_result), - target=item["domain"], - report_type=HumbleReporter.MISSING_SECURITY_HEADER, - additional_data={ - "message_en": item["info"], - }, - timestamp=task_result["created_at"], - ) - ) + result.append( + Report( + top_level_target=get_top_level_target(task_result), + target=get_target_url(task_result), + report_type=HumbleReporter.MISSING_SECURITY_HEADERS, + additional_data={ + "message_data": task_result["result"]["message_data"], + "messages": task_result["result"]["messages"], + }, + timestamp=task_result["created_at"], + ) + ) return result @staticmethod @@ -49,20 +48,15 @@ def get_email_template_fragments() -> List[ReportEmailTemplateFragment]: ), ] - @staticmethod - def get_scoring_rules() -> Dict[ReportType, Callable[[Report], List[int]]]: - """See the docstring in the parent class.""" - return {HumbleReporter.MISSING_SECURITY_HEADER: lambda report: [get_domain_score(report.target)]} - @staticmethod def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]: """See the docstring in the Reporter class.""" return { - HumbleReporter.MISSING_SECURITY_HEADER: lambda report: Reporter.dict_to_tuple( + HumbleReporter.MISSING_SECURITY_HEADERS: lambda report: Reporter.dict_to_tuple( { "type": report.report_type, - "target": get_domain_normal_form(report.target), - "message": report.additional_data["message_en"], + "target": get_url_normal_form(report.target), + "messages": tuple(report.additional_data["messages"]), # type: ignore } ) } diff --git a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 index 58cf1aa6b..51490257d 100644 --- a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 +++ b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 @@ -1,12 +1,14 @@ -{% if "missing_security_header" in data.contains_type %} -
  • {% trans %}We identified that not all possible security headers are set correctly:{% endtrans %} +{% if "missing_security_headers" in data.contains_type %} +
  • {% trans %}We identified that the following security headers are set correctly:{% endtrans %}
      {% for report in data.reports %} - {% if report.report_type == "missing_security_header" %} -
    • - {{ report.target }}: {{ _(report.additional_data.message_en) }} - {{ report_meta(report) }} -
      • + {% if report.report_type == "missing_security_headers" %} + {% for message in report.additional_data.message_data %} +
    • + {{ report.target }}: {{ _(message.category) }}: {{ ', '.join(message.problems) }} +
      • + {% endfor %} + {{ report_meta(report) }} {% endif %} {% endfor %}
    diff --git a/artemis/reporting/severity.py b/artemis/reporting/severity.py index c9a7dfa32..a21c45655 100644 --- a/artemis/reporting/severity.py +++ b/artemis/reporting/severity.py @@ -52,4 +52,5 @@ class Severity(str, Enum): ReportType("exposed_php_var_dump"): Severity.LOW, ReportType("exposed_phpinfo"): Severity.LOW, ReportType("nuclei_exposed_panel"): Severity.LOW, + ReportType("missing_security_headers"): Severity.LOW, } diff --git a/test/modules/test_humble.py b/test/modules/test_humble.py index f9416bb36..49b772035 100644 --- a/test/modules/test_humble.py +++ b/test/modules/test_humble.py @@ -7,35 +7,30 @@ class TestProcessJsonData(unittest.TestCase): def test_process_json_data_with_valid_input(self) -> None: # Setup input_data = { - "[0. Info]": { - "Date": "1970/01/01 - 12:12:12", - "URL": "https://test.tld" - }, - "[1. Missing HTTP Security Headers]": [ - "Cache-Control", - "Clear-Site-Data", - "Cross-Origin-Embedder-Policy" - ], + "[0. Info]": {"Date": "1970/01/01 - 12:12:12", "URL": "https://test.tld"}, + "[1. Missing HTTP Security Headers]": ["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"], "[2. Fingerprint HTTP Response Headers]": [ "Server", ], - "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": [ - "X-XSS-Protection (Unsafe Value)" - ], - "[4. Empty HTTP Response Headers Values]": [ - "Content-Security-Policy" - ], + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]": ["X-XSS-Protection (Unsafe Value)"], + "[4. Empty HTTP Response Headers Values]": ["Content-Security-Policy"], "[5. Browser Compatibility for Enabled HTTP Security Headers]": { "X-XSS-Protection": "https://caniuse.com/?search=X-XSS-Protection" - } + }, } # Exercise result = humble.process_json_data(input_data) # Verify - expected_result = ["Missing http security headers:", "Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"] + expected_result = [ + "Missing http security headers:", + "Cache-Control", + "Clear-Site-Data", + "Cross-Origin-Embedder-Policy", + ] self.assertEqual(result, expected_result) + if __name__ == "__main__": unittest.main() From a13a55a10d327c0da98fc29b37b60ad204784331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:42:27 +0100 Subject: [PATCH 04/19] ordering --- docker-compose.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 585d4334d..84dcb5605 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -172,6 +172,16 @@ services: restart: always volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] + karton-humble: + build: + context: . + dockerfile: docker/Dockerfile + command: "python3 -m artemis.modules.humble" + depends_on: [karton-logger] + env_file: .env + restart: always + volumes: [ "./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/" ] + karton-identifier: build: context: . @@ -367,16 +377,6 @@ services: restart: always volumes: ["./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/"] - karton-humble: - build: - context: . - dockerfile: docker/Dockerfile - volumes: [ "./docker/karton.ini:/etc/karton/karton.ini", "${DOCKER_COMPOSE_ADDITIONAL_SHARED_DIRECTORY:-./shared}:/shared/" ] - depends_on: [ karton-logger, redis ] - env_file: .env - restart: always - command: "python3 -m artemis.modules.humble" - volumes: data-mongodb: data-redis: From fa3a6808ea422a342c5ac409bc7f0c9cc1e6680d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:46:59 +0100 Subject: [PATCH 05/19] . --- .../template_missing_security_header.jinja2 | 2 +- .../translations/en_US/LC_MESSAGES/messages.po | 10 ++++++++++ .../translations/pl_PL/LC_MESSAGES/additional.po | 7 +++++++ .../translations/pl_PL/LC_MESSAGES/messages.po | 15 +++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po create mode 100644 artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po create mode 100644 artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po diff --git a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 index 51490257d..ab1651114 100644 --- a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 +++ b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 @@ -1,5 +1,5 @@ {% if "missing_security_headers" in data.contains_type %} -
  • {% trans %}We identified that the following security headers are set correctly:{% endtrans %} +
  • {% trans %}We identified that the following security headers are not set correctly:{% endtrans %}
      {% for report in data.reports %} {% if report.report_type == "missing_security_headers" %} diff --git a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po new file mode 100644 index 000000000..a39663ecb --- /dev/null +++ b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po @@ -0,0 +1,10 @@ +#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:2 +msgid "We identified that the following security headers are not set correctly:" +msgstr "" + +#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 +msgid "" +"Please verify the configuration, and, if a security header is missing, " +"change it. Security headers can have a deep impact in protecting your " +"application against attacks." +msgstr "" diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po new file mode 100644 index 000000000..c96b53b1f --- /dev/null +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po @@ -0,0 +1,7 @@ +# This is a separate file so that pybabel doesn't mark it as obsolete because it doesn't +# see the original strings in our code (they come from the humble library). + +msgid "" +"Missing http security headers" +msgstr "" +"Brakujące nagłówki HTTP zwiększajace bezpieczeństwo" diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po new file mode 100644 index 000000000..10b5b6aae --- /dev/null +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po @@ -0,0 +1,15 @@ +#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:2 +msgid "We identified that the following security headers are not set correctly:" +msgstr "" +"Wykryto, że następujące nagłówki HTTP zwiększające bezpieczeństwo nie są " +"ustawione poprawnie:" + +#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 +msgid "" +"Please verify the configuration, and, if a security header is missing, " +"change it. Security headers can have a deep impact in protecting your " +"application against attacks." +msgstr "" +"Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, " +"jeśli nie jest skonfigurowany poprawnie. Poprawna konfiguracja powyższych" +" nagłówków zwiększy bezpieczeństwo aplikacji." From 66b524211cf65c8640dec08c91e13cb2ae23873d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:47:31 +0100 Subject: [PATCH 06/19] linter --- artemis/reporting/modules/humble/reporter.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py index b169ab1c6..4f8f34631 100644 --- a/artemis/reporting/modules/humble/reporter.py +++ b/artemis/reporting/modules/humble/reporter.py @@ -2,11 +2,7 @@ from typing import Any, Callable, Dict, List from artemis.reporting.base.language import Language -from artemis.reporting.base.normal_form import ( - NormalForm, - get_url_normal_form, - get_url_score, -) +from artemis.reporting.base.normal_form import NormalForm, get_url_normal_form from artemis.reporting.base.report import Report from artemis.reporting.base.report_type import ReportType from artemis.reporting.base.reporter import Reporter From a67ca2440ac77c4f5c2a85212d1389e58ce65ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:48:35 +0100 Subject: [PATCH 07/19] grammar --- .../modules/humble/template_missing_security_header.jinja2 | 2 +- .../modules/humble/translations/en_US/LC_MESSAGES/messages.po | 2 +- .../modules/humble/translations/pl_PL/LC_MESSAGES/messages.po | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 index ab1651114..b8d83c4aa 100644 --- a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 +++ b/artemis/reporting/modules/humble/template_missing_security_header.jinja2 @@ -15,7 +15,7 @@

      {% trans trimmed %} Please verify the configuration, and, if a security header is missing, change it. Security - headers can have a deep impact in protecting your application against attacks. + headers can have a deep impact on protecting your application against attacks. {% endtrans %}

      diff --git a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po index a39663ecb..e4fbffe07 100644 --- a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po @@ -5,6 +5,6 @@ msgstr "" #: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " -"change it. Security headers can have a deep impact in protecting your " +"change it. Security headers can have a deep impact on protecting your " "application against attacks." msgstr "" diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po index 10b5b6aae..6acb3f8f6 100644 --- a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgstr "" #: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " -"change it. Security headers can have a deep impact in protecting your " +"change it. Security headers can have a deep impact on protecting your " "application against attacks." msgstr "" "Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, " From 93f13be0c1d142e32d629bda8aba4bc678407064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:49:28 +0100 Subject: [PATCH 08/19] ogonki --- .../modules/humble/translations/pl_PL/LC_MESSAGES/additional.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po index c96b53b1f..326ff32ac 100644 --- a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/additional.po @@ -4,4 +4,4 @@ msgid "" "Missing http security headers" msgstr "" -"Brakujące nagłówki HTTP zwiększajace bezpieczeństwo" +"Brakujące nagłówki HTTP zwiększające bezpieczeństwo" From 5d9ffbf2cb8f909c4d4d9248e90a7e4b700bb2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:54:29 +0100 Subject: [PATCH 09/19] . --- artemis/reporting/modules/humble/reporter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py index 4f8f34631..e90d84423 100644 --- a/artemis/reporting/modules/humble/reporter.py +++ b/artemis/reporting/modules/humble/reporter.py @@ -21,8 +21,7 @@ def create_reports(task_result: Dict[str, Any], language: Language) -> List[Repo if not isinstance(task_result["result"], dict): return [] - result = [] - result.append( + return [ Report( top_level_target=get_top_level_target(task_result), target=get_target_url(task_result), @@ -33,8 +32,7 @@ def create_reports(task_result: Dict[str, Any], language: Language) -> List[Repo }, timestamp=task_result["created_at"], ) - ) - return result + ] @staticmethod def get_email_template_fragments() -> List[ReportEmailTemplateFragment]: From a2e937436b95d15259be5626c20b24c92a9c353c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:55:02 +0100 Subject: [PATCH 10/19] plural --- artemis/reporting/modules/humble/reporter.py | 2 +- ...y_header.jinja2 => template_missing_security_headers.jinja2} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename artemis/reporting/modules/humble/{template_missing_security_header.jinja2 => template_missing_security_headers.jinja2} (100%) diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py index e90d84423..8529348f7 100644 --- a/artemis/reporting/modules/humble/reporter.py +++ b/artemis/reporting/modules/humble/reporter.py @@ -38,7 +38,7 @@ def create_reports(task_result: Dict[str, Any], language: Language) -> List[Repo def get_email_template_fragments() -> List[ReportEmailTemplateFragment]: return [ ReportEmailTemplateFragment.from_file( - str(Path(__file__).parents[0] / "template_missing_security_header.jinja2"), priority=2 + str(Path(__file__).parents[0] / "template_missing_security_headers.jinja2"), priority=2 ), ] diff --git a/artemis/reporting/modules/humble/template_missing_security_header.jinja2 b/artemis/reporting/modules/humble/template_missing_security_headers.jinja2 similarity index 100% rename from artemis/reporting/modules/humble/template_missing_security_header.jinja2 rename to artemis/reporting/modules/humble/template_missing_security_headers.jinja2 From e40102a377a2ca24743e9f658947cb157eeb4e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 12:56:30 +0100 Subject: [PATCH 11/19] updating translations --- .../modules/humble/translations/en_US/LC_MESSAGES/messages.po | 4 ++-- .../modules/humble/translations/pl_PL/LC_MESSAGES/messages.po | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po index e4fbffe07..ec9311c3f 100644 --- a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po @@ -1,8 +1,8 @@ -#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:2 +#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2 msgid "We identified that the following security headers are not set correctly:" msgstr "" -#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 +#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " "change it. Security headers can have a deep impact on protecting your " diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po index 6acb3f8f6..895c5fc4d 100644 --- a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po @@ -1,10 +1,10 @@ -#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:2 +#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:2 msgid "We identified that the following security headers are not set correctly:" msgstr "" "Wykryto, że następujące nagłówki HTTP zwiększające bezpieczeństwo nie są " "ustawione poprawnie:" -#: artemis/reporting/modules/humble/template_missing_security_header.jinja2:16 +#: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " "change it. Security headers can have a deep impact on protecting your " From 9657fbd6ad1e750437197d4a496a5e252f8c6ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 14:21:32 +0100 Subject: [PATCH 12/19] test fix --- test/modules/test_humble.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/modules/test_humble.py b/test/modules/test_humble.py index 49b772035..81b701d42 100644 --- a/test/modules/test_humble.py +++ b/test/modules/test_humble.py @@ -24,10 +24,10 @@ def test_process_json_data_with_valid_input(self) -> None: # Verify expected_result = [ - "Missing http security headers:", - "Cache-Control", - "Clear-Site-Data", - "Cross-Origin-Embedder-Policy", + humble.Message( + category="Missing http security headers", + problems=["Cache-Control", "Clear-Site-Data", "Cross-Origin-Embedder-Policy"], + ) ] self.assertEqual(result, expected_result) From fd87ee19eeaf92573ba9bc6a9cf73710231c5203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 15:24:12 +0100 Subject: [PATCH 13/19] Filtering to report only most important headers --- artemis/config.py | 11 +++++++++++ artemis/reporting/base/reporter.py | 12 ++++++++++-- artemis/reporting/modules/humble/reporter.py | 18 +++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/artemis/config.py b/artemis/config.py index e68a1900e..a71a205d0 100644 --- a/artemis/config.py +++ b/artemis/config.py @@ -229,6 +229,17 @@ class Gau: "Additional command-line options that will be passed to gau (https://github.com/lc/gau).", ] = get_config("GAU_ADDITIONAL_OPTIONS", default="", cast=decouple.Csv(str, delimiter=" ")) + class Humble: + HUMBLE_HEADERS_TO_REPORT: Annotated[ + List[str], + "The list of headers that are considered more important and will be mentioned in the generated text " + "reports (all of the missing headers will be visible in the UI).", + ] = get_config( + "HUMBLE_HEADERS_TO_REPORT", + default=",".join(["Content-Security-Policy", "Strict-Transport-Security", "X-Content-Type-Options"]), + cast=decouple.Csv(str, delimiter=","), + ) + class Nuclei: NUCLEI_CHECK_TEMPLATE_LIST: Annotated[ bool, diff --git a/artemis/reporting/base/reporter.py b/artemis/reporting/base/reporter.py index 9fab46419..911ac3295 100644 --- a/artemis/reporting/base/reporter.py +++ b/artemis/reporting/base/reporter.py @@ -65,8 +65,16 @@ def get_scoring_rules(cls) -> Dict[ReportType, Callable[[Report], List[int]]]: return {report_type: Reporter.default_scoring_rule for report_type in cls.get_report_types()} @staticmethod - def dict_to_tuple(d: Dict[str, str]) -> Tuple[Tuple[str, str], ...]: - return tuple(d.items()) + def dict_to_tuple(d: Dict[str, Any]) -> Tuple[Any, ...]: + result = [] + for key, value in d.items(): + if isinstance(value, dict): + result.append((key, Reporter.dict_to_tuple(value))) + elif isinstance(value, list): + result.append((key, tuple(value))) + else: + result.append((key, value)) + return tuple(result) @staticmethod def default_scoring_rule(report: Report) -> List[int]: diff --git a/artemis/reporting/modules/humble/reporter.py b/artemis/reporting/modules/humble/reporter.py index 8529348f7..9df118a78 100644 --- a/artemis/reporting/modules/humble/reporter.py +++ b/artemis/reporting/modules/humble/reporter.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List +from artemis.config import Config from artemis.reporting.base.language import Language from artemis.reporting.base.normal_form import NormalForm, get_url_normal_form from artemis.reporting.base.report import Report @@ -27,8 +28,7 @@ def create_reports(task_result: Dict[str, Any], language: Language) -> List[Repo target=get_target_url(task_result), report_type=HumbleReporter.MISSING_SECURITY_HEADERS, additional_data={ - "message_data": task_result["result"]["message_data"], - "messages": task_result["result"]["messages"], + "message_data": HumbleReporter._filter_message_data(task_result["result"]["message_data"]), }, timestamp=task_result["created_at"], ) @@ -50,7 +50,19 @@ def get_normal_form_rules() -> Dict[ReportType, Callable[[Report], NormalForm]]: { "type": report.report_type, "target": get_url_normal_form(report.target), - "messages": tuple(report.additional_data["messages"]), # type: ignore + "message_data": [Reporter.dict_to_tuple(item) for item in report.additional_data["message_data"]], } ) } + + @staticmethod + def _filter_message_data(message_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + for item in message_data: + if item["category"] == "Missing http security headers": + problems = sorted(set(item["problems"]) & set(Config.Modules.Humble.HUMBLE_HEADERS_TO_REPORT)) + if problems: + result.append({"category": item["category"], "problems": problems}) + else: + result.append(item) + return result From 4d2e701a3be97602bb0620188978b03866935c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Sun, 14 Jan 2024 16:20:36 +0100 Subject: [PATCH 14/19] Test fix --- test/modules/test_joomla_scanner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/modules/test_joomla_scanner.py b/test/modules/test_joomla_scanner.py index 30549a273..16795df5c 100644 --- a/test/modules/test_joomla_scanner.py +++ b/test/modules/test_joomla_scanner.py @@ -1,9 +1,10 @@ import datetime import unittest.mock -from test.base import ArtemisModuleTestCase +from test.base import ArtemisModuleTestCase, KartonBackendMockWithRedis from freezegun import freeze_time from karton.core import Task +from karton.core.test import ConfigMock from artemis.binds import TaskStatus, TaskType, WebApplication from artemis.modules.joomla_scanner import JoomlaScanner @@ -47,7 +48,12 @@ class JoomlaScannerTest(ArtemisModuleTestCase): @freeze_time("2023-02-21") def test_is_newer_version_available(self) -> None: - with unittest.mock.patch("yaml.load_all", return_value=self.endoflife_data): + with unittest.mock.patch("yaml.load_all", return_value=self.endoflife_data.__iter__()): + # Recreate the karton with mocked endoflife data + self.karton = self.karton_class( # type: ignore + config=ConfigMock(), backend=KartonBackendMockWithRedis(), db=self.mock_db # type: ignore + ) + self.assertTrue(self.karton.is_version_obsolete("2.8.6")) self.assertTrue(self.karton.is_version_obsolete("2.99999.99999")) From b9e39682413d5354f564c6d9b834cc42a77f8879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Tue, 16 Jan 2024 10:15:26 +0100 Subject: [PATCH 15/19] review fix 1 --- artemis/modules/humble.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/artemis/modules/humble.py b/artemis/modules/humble.py index ed42231be..bd9a13d7e 100644 --- a/artemis/modules/humble.py +++ b/artemis/modules/humble.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import dataclasses import json +import os import subprocess from typing import Any, Dict, List @@ -121,7 +122,7 @@ def run(self, current_task: Task) -> None: data_str = open(filename, "r").read() # cleanup file - subprocess.run(["rm", filename]) + os.unlink(filename) # Check if the input string is empty if data_str.strip(): From 529e37ca5efb31c0ece2a502cf0c0175150b0853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Tue, 16 Jan 2024 10:16:49 +0100 Subject: [PATCH 16/19] review fix part 2 --- artemis/modules/humble.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/artemis/modules/humble.py b/artemis/modules/humble.py index bd9a13d7e..1f1310c84 100644 --- a/artemis/modules/humble.py +++ b/artemis/modules/humble.py @@ -30,31 +30,29 @@ def process_json_data(result: Dict[str, Any]) -> List[Message]: # Split the key to extract the relevant part key_parts = key.replace("[", "").replace("]", "").split(". ") + if key in [ + "[2. Fingerprint HTTP Response Headers]", + "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]", + "[4. Empty HTTP Response Headers Values]", + "[5. Browser Compatibility for Enabled HTTP Security Headers]", + ]: + continue + # Check if the key has the expected structure if len(key_parts) >= 2: category = key_parts[1].capitalize() # Check if the key is not in the excluded categories and there are relevant values - if ( - category.lower() != "info" - and key - not in [ - "[2. Fingerprint HTTP Response Headers]", - "[3. Deprecated HTTP Response Headers/Protocols and Insecure Values]", - "[4. Empty HTTP Response Headers Values]", - "[5. Browser Compatibility for Enabled HTTP Security Headers]", - ] - and ( - isinstance(value, dict) - or ( - isinstance(value, list) - and any( - subvalue - for subvalue in value - if subvalue - and subvalue - not in ["Nothing to report, all seems OK!", "No HTTP security headers are enabled."] - ) + if category.lower() != "info" and ( + isinstance(value, dict) + or ( + isinstance(value, list) + and any( + subvalue + for subvalue in value + if subvalue + and subvalue + not in ["Nothing to report, all seems OK!", "No HTTP security headers are enabled."] ) ) ): From 9f274210c3f95a084414d6d22783833d4b5e37a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Tue, 16 Jan 2024 10:17:33 +0100 Subject: [PATCH 17/19] review fix part 3 --- artemis/modules/humble.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/artemis/modules/humble.py b/artemis/modules/humble.py index 1f1310c84..f354b66bb 100644 --- a/artemis/modules/humble.py +++ b/artemis/modules/humble.py @@ -43,19 +43,7 @@ def process_json_data(result: Dict[str, Any]) -> List[Message]: category = key_parts[1].capitalize() # Check if the key is not in the excluded categories and there are relevant values - if category.lower() != "info" and ( - isinstance(value, dict) - or ( - isinstance(value, list) - and any( - subvalue - for subvalue in value - if subvalue - and subvalue - not in ["Nothing to report, all seems OK!", "No HTTP security headers are enabled."] - ) - ) - ): + if category.lower() != "info": # If the value is a dictionary, iterate through subkey-value pairs if isinstance(value, dict): for subkey, subvalue in value.items(): From ac2458ef088c324041d4c6fc3499472a5edc6cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Tue, 16 Jan 2024 10:44:22 +0100 Subject: [PATCH 18/19] . --- .../modules/humble/template_missing_security_headers.jinja2 | 2 +- .../modules/humble/translations/en_US/LC_MESSAGES/messages.po | 2 +- .../modules/humble/translations/pl_PL/LC_MESSAGES/messages.po | 2 +- .../modules/nuclei/translations/nuclei_messages/pl_PL.py | 1 + env.example | 3 --- 5 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 env.example diff --git a/artemis/reporting/modules/humble/template_missing_security_headers.jinja2 b/artemis/reporting/modules/humble/template_missing_security_headers.jinja2 index b8d83c4aa..4b8d482ae 100644 --- a/artemis/reporting/modules/humble/template_missing_security_headers.jinja2 +++ b/artemis/reporting/modules/humble/template_missing_security_headers.jinja2 @@ -14,7 +14,7 @@

    {% trans trimmed %} - Please verify the configuration, and, if a security header is missing, change it. Security + Please verify the configuration, and, if a security header is missing, add it. Security headers can have a deep impact on protecting your application against attacks. {% endtrans %}

    diff --git a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po index ec9311c3f..12b8a76fa 100644 --- a/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/en_US/LC_MESSAGES/messages.po @@ -5,6 +5,6 @@ msgstr "" #: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " -"change it. Security headers can have a deep impact on protecting your " +"add it. Security headers can have a deep impact on protecting your " "application against attacks." msgstr "" diff --git a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po index 895c5fc4d..cf620def2 100644 --- a/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po +++ b/artemis/reporting/modules/humble/translations/pl_PL/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgstr "" #: artemis/reporting/modules/humble/template_missing_security_headers.jinja2:16 msgid "" "Please verify the configuration, and, if a security header is missing, " -"change it. Security headers can have a deep impact on protecting your " +"add it. Security headers can have a deep impact on protecting your " "application against attacks." msgstr "" "Rekomendujemy weryfikację konfiguracji i zmianę lub ustawienie nagłówka, " diff --git a/artemis/reporting/modules/nuclei/translations/nuclei_messages/pl_PL.py b/artemis/reporting/modules/nuclei/translations/nuclei_messages/pl_PL.py index 1beb253bb..b6d90cc14 100644 --- a/artemis/reporting/modules/nuclei/translations/nuclei_messages/pl_PL.py +++ b/artemis/reporting/modules/nuclei/translations/nuclei_messages/pl_PL.py @@ -584,6 +584,7 @@ "Execution After Redirect happens when after emitting a Location header that redirects the user, some other code is executed. This may lead to data leak or application compromise.": "Wykryto podatność Execution After Redirect, czyli sytuację, gdy serwer, pomimo przekierowania użytkownika na inny adres, kontynuuje wykonanie skryptu. Może to doprowadzić do wycieku wrażliwych danych lub uzyskania przez atakującego nieuprawnionego dostępu do aplikacji.", "[no description] http/misconfiguration/django-debug-detect.yaml": "Wykryto system Django w konfiguracji debug. Upublicznienie systemu w takiej konfiguracji może umożliwić atakującemu poznanie informacji na temat działania aplikacji lub jej konfiguracji.", "DOMPDF Configuration page was detected, which contains paths, library versions and other potentially sensitive information": "Wykryto stronę konfiguracyjną DOMPDF, która zawiera ścieżki, wersje zainstalowanego oprogramowania i inne potencjalnie wrażliwe informacje.", + "This check detects if there are any active content loaded over HTTP instead of HTTPS.": "Wykryto, że zasoby takie jak skrypty są ładowane za pomocą nieszyfrowanego połączenia. Może to umożliwić atakującemu ich podmianę, a w konsekwencji zmianę wyglądu lub zachowania strony.", "OwnCloud is susceptible to the Installation page exposure due to misconfiguration.": "wykryto, że panel instalacyjny narzędzia OwnCloud jest publicznie dostępny, co może umożliwić atakującemu nieuprawniony dostęp do systemu.", "phpMyAdmin panel was detected.": "wykryto panel logowania narzędzia phpMyAdmin.", "WordPress login panel was detected.": "wykryto panel logowania systemu WordPress.", diff --git a/env.example b/env.example deleted file mode 100644 index 8343816c8..000000000 --- a/env.example +++ /dev/null @@ -1,3 +0,0 @@ -DB_CONN_STR=mongodb://root:root@db -REDIS_CONN_STR=redis://redis:6379/1 -CUSTOM_USER_AGENT= From 36aa2810dfd2de94e91a07c6f35e407e792336a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Zaj=C4=85c?= Date: Tue, 16 Jan 2024 10:45:24 +0100 Subject: [PATCH 19/19] . --- env.example | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 env.example diff --git a/env.example b/env.example new file mode 100644 index 000000000..8343816c8 --- /dev/null +++ b/env.example @@ -0,0 +1,3 @@ +DB_CONN_STR=mongodb://root:root@db +REDIS_CONN_STR=redis://redis:6379/1 +CUSTOM_USER_AGENT=