ENH: Added Log4shell shadowserver feed

Fixes #2131 Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>
certtools · Feb 4, 2022 · bc9abfa · bc9abfa
1 parent 8d1c926
commit bc9abfa
Show file tree

Hide file tree

Showing 4 changed files with 146 additions and 0 deletions.
diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py
@@ -2970,6 +2970,33 @@ def scan_exchange_identifier(field):
     }
 }
 
+# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-log4j-servers-special-report/
+vulnerable_log4shell = {
+    'required_fields': [
+        ('time.source', 'timestamp', add_UTC_to_timestamp),
+        ('source.ip', 'ip'),
+        ('source.port', 'port')
+    ],
+    'optional_fields': [
+        ('protocol.transport', 'protocol'),
+        ('source.asn', 'asn', invalidate_zero),
+        ('source.geolocation.region', 'region'),
+        ('source.geolocation.city', 'city'),
+        ('source.reverse_dns', 'hostname'),
+        ('extra.', 'naics', invalidate_zero),
+        ('extra.', 'sector', validate_to_none),
+        ('extra.', 'tag', validate_to_none),
+        ('extra.', 'public_source', validate_to_none),
+        ('extra.', 'status', validate_to_none),
+        ('extra.', 'method', validate_to_none),
+    ],
+    'constant_fields': {
+        'classification.taxonomy': 'vulnerable',
+        'classification.type': 'vulnerable-system',
+        'classification.identifier': 'log4shell',
+    }
+}
+
 mapping = (
     # feed name, file name, function
     ('Accessible-ADB', 'scan_adb', accessible_adb),
@@ -3059,6 +3086,7 @@ def scan_exchange_identifier(field):
     ('Vulnerable-HTTP', 'scan_http_vulnerable', accessible_vulnerable_http),
     ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange),
     ('Vulnerable-SMTP', 'scan_smtp_vulnerable', vulnerable_smtp),
+    ('Vulnerable-Log4J', 'scan_log4shell_vulnerable', vulnerable_log4shell),
 )
 
 feedname_mapping = {feedname: function for feedname, filename, function in mapping}

diff --git a/intelmq/tests/bots/parsers/shadowserver/test_log4shell.py b/intelmq/tests/bots/parsers/shadowserver/test_log4shell.py
@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: 2022 CERT.at GmbH <waldbauer@cert.at>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+with open(os.path.join(os.path.dirname(__file__),
+                       'testdata/scan_log4shell_vulnerable.csv')) as handle:
+    EXAMPLE_FILE = handle.read()
+EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
+
+EXAMPLE_REPORT = {'feed.name': 'Vulnerable Log4J',
+                  "raw": utils.base64_encode(EXAMPLE_FILE),
+                  "__type": "Report",
+                  "time.observation": "2021-12-13T09:00:00+00:00",
+                  "extra.file_name": "2021-12-13-scan_log4shell_vulnerable-test-test.csv",
+                  }
+EVENTS = [{'__type': 'Event',
+           'feed.name': 'Vulnerable Log4J',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'classification.identifier': 'log4shell',
+           'extra.geo': 'IR',
+           'extra.method': 'dns',
+           'extra.public_source': 'alphastrike.io',
+           'extra.status': 'vulnerable',
+           'extra.tag': 'cve-2021-44228',
+           'protocol.transport': 'tcp',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[1]])),
+           'source.ip': '198.51.100.147',
+           'source.asn': 41881,
+           'source.geolocation.city': 'TEHRAN',
+           'source.geolocation.region': 'TEHRAN',
+           'source.port': 80,
+           'time.observation': '2021-12-13T13:58:00+00:00',
+           'time.source': '2021-12-13T13:58:00+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'Vulnerable Log4J',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'classification.identifier': 'log4shell',
+           'extra.naics': 454110,
+           'extra.geo': 'US',
+           'extra.method': 'dns',
+           'extra.public_source': 'alphastrike.io',
+           'extra.status': 'vulnerable',
+           'extra.tag': 'cve-2021-44228',
+           'protocol.transport': 'tcp',
+           'extra.sector': 'Retail Trade',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[2]])),
+           'source.ip': '198.51.100.147',
+           'source.asn': 14618,
+           'source.geolocation.city': 'ASHBURN',
+           'source.geolocation.region': 'VIRGINIA',
+           'source.port': 443,
+           'source.reverse_dns': '198-51-100-147.example.net',
+           'time.observation': '2021-12-13T13:58:00+00:00',
+           'time.source': '2021-12-13T13:58:00+00:00'},
+          {'__type': 'Event',
+           'feed.name': 'Vulnerable Log4J',
+           'classification.taxonomy': 'vulnerable',
+           'classification.type': 'vulnerable-system',
+           'classification.identifier': 'log4shell',
+           'extra.naics': 454110,
+           'extra.geo': 'US',
+           'extra.method': 'dns',
+           'extra.public_source': 'alphastrike.io',
+           'extra.status': 'vulnerable',
+           'extra.tag': 'cve-2021-44228',
+           'protocol.transport': 'tcp',
+           'extra.sector': 'Retail Trade',
+           'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+                                                 EXAMPLE_LINES[3]])),
+           'source.ip': '198.51.100.147',
+           'source.asn': 14618,
+           'source.geolocation.city': 'ASHBURN',
+           'source.geolocation.region': 'VIRGINIA',
+           'source.port': 8080,
+           'source.reverse_dns': '198-51-100-147.example.net',
+           'time.observation': '2021-12-13T13:58:00+00:00',
+           'time.source': '2021-12-13T13:58:00+00:00'}
+          ]
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+    """
+    A TestCase for a ShadowserverParserBot.
+    """
+
+    @classmethod
+    def set_bot(cls):
+        cls.bot_reference = ShadowserverParserBot
+        cls.default_input_message = EXAMPLE_REPORT
+
+    def test_event(self):
+        """ Test if correct Event has been produced. """
+        self.run_bot()
+        for i, EVENT in enumerate(EVENTS):
+            self.assertMessageEqual(i, EVENT)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    unittest.main()
diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_log4shell_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_log4shell_vulnerable.csv
@@ -0,0 +1,4 @@
+"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","method"
+"2021-12-13 13:58:00",198.51.100.147,80,tcp,41881,IR,TEHRAN,TEHRAN,,,,cve-2021-44228,alphastrike.io,vulnerable,dns
+"2021-12-13 13:58:00",198.51.100.147,443,tcp,14618,US,VIRGINIA,ASHBURN,198-51-100-147.example.net,454110,"Retail Trade",cve-2021-44228,alphastrike.io,vulnerable,dns
+"2021-12-13 13:58:00",198.51.100.147,8080,tcp,14618,US,VIRGINIA,ASHBURN,198-51-100-147.example.net,454110,"Retail Trade",cve-2021-44228,alphastrike.io,vulnerable,dns
diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_log4shell_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_log4shell_vulnerable.csv.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2022 CERT.at GmbH
+SPDX-License-Identifier: AGPL-3.0-or-later