bug: fix ctip parser handling JSON Payload content

Handle Payload field with non-base64-encoded JSON content and numbered dictionaries
certtools · Jun 2, 2022 · 9a5d17f · 9a5d17f
1 parent cef39c1
commit 9a5d17f
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -58,6 +58,7 @@ CHANGELOG
 - `intelmq.bots.parsers.microsoft.parser_ctip`:
   - New parameter `overwrite` (PR#2112 by Sebastian Wagner, fixes #2022).
   - Fix handling of field `Payload.domain` if it contains the same IP address as `Payload.serverIp` (PR#2144 by Mikk Margus Möll and Sebastian Wagner).
+  - Handle Payload field with non-base64-encoded JSON content and numbered dictionaries (PR#2193 by Sebastian Wagner)
 - `intelmq.bot.parsers.shodan.parser` (PR#2117 by Mikk Margus Möll):
   - Instead of keeping track of `extra.ftp.<something>.parameters`, FTP parameters are collected together into `extra.ftp.features` as a list of said features, reducing field count.
   - Shodan field `rsync.modules` is collected.

diff --git a/intelmq/bots/parsers/microsoft/parser_ctip.py b/intelmq/bots/parsers/microsoft/parser_ctip.py
@@ -59,7 +59,7 @@
            "CustomField4": "",
            "CustomField5": ""
          },
-         "Payload": base64 encoded json
+         "Payload": base64 encoded json with meaningful dictionary keys or JSON-string with numbered dictionary keys
        }
 
 """
@@ -263,14 +263,23 @@ def parse_azure(self, line, report):
 
         for key, value in line.copy().items():
             if key == 'Payload':
+                # empty
                 if value == 'AA==':  # NULL
                     del line[key]
                     continue
-                try:
-                    value = json.loads(utils.base64_decode(value))
-                    # continue unpacking in next loop
-                except json.decoder.JSONDecodeError:
-                    line[key] = utils.base64_decode(value)
+
+                # JSON string
+                if value.startswith('{'):
+                    for payload_key, payload_value in json.loads(value).items():
+                        event[f'extra.payload.{payload_key}'] = payload_value
+                    del line[key]
+                else:
+                    # base64-encoded JSON
+                    try:
+                        value = json.loads(utils.base64_decode(value))
+                        # continue unpacking in next loop
+                    except json.decoder.JSONDecodeError:
+                        line[key] = utils.base64_decode(value)
             elif key == 'TLP' and value.lower() == 'unknown':
                 del line[key]
             if isinstance(value, dict):

diff --git a/intelmq/etc/feeds.yaml b/intelmq/etc/feeds.yaml
@@ -723,21 +723,21 @@ providers:
         services on the router or tried to gain access to them. The list also
         contains a list of tags for each address which
         indicate what behaviour of the address was observed.
-        
+
         The Turris Greylist feed provides PGP signatures for the provided files.
         You will need to import the public PGP key from the linked documentation
         page, currently available at
         https://pgp.mit.edu/pks/lookup?op=vindex&search=0x10876666
         or from below.
         See the URL Fetcher Collector documentation for more information on
         PGP signature verification.
-        
+
         PGP Public key:
         ```
         -----BEGIN PGP PUBLIC KEY BLOCK-----
         Version: SKS 1.1.6
         Comment: Hostname: pgp.mit.edu
-        
+
         mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0
         o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t
         3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40
@@ -1756,7 +1756,7 @@ providers:
         parser:
           module: intelmq.bots.parsers.microsoft.parser_ctip
           parameters:
-      revision: 2020-05-29
+      revision: 2022-06-01
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange http://www.dcuctip.com/
       public: false
     CTIP C2 via Azure:
@@ -1887,10 +1887,10 @@ providers:
                       listen 443 ssl http2;
                       server_name [your host name];
                       client_max_body_size 50M;
-                      
+
                       ssl_certificate [path to your key];
                       ssl_certificate_key [path to your certificate];
-                      
+
                       location /[your private url] {
                            if ($http_authorization != '[your private password]') {
                                return 403;

diff --git a/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt b/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt
@@ -3,3 +3,4 @@
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiMTAuMC4wLjEiLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132990083418030000,"DateTimeReceivedUtcTxt":"Wednesday June 01 2022 13:33:13.3713","Malware":"Malware","ThreatCode":"B00-Leet","ThreatConfidence":"High","TotalEncounters":137,"TLP":"Green","SourceIp":"10.0.0.15","SourcePort":10000,"DestinationIp":"10.0.0.2","DestinationPort":443,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"My ISP","SourceIpCountryCode":"DE","SourceIpRegion":"Saarland","SourceIpCity":"Saarbrücken","SourceIpPostalCode":"66111","SourceIpLatitude":49.2367,"SourceIpLongitude":6.9794,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cable/DSL","SourceIpv4Int":167772175},"HttpInfo":{"HttpHost":"example.com","HttpRequest":"/index.php","HttpMethod":"POST","HttpReferrer":"","HttpUserAgent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36","HttpVersion":"HTTP/1.1"},"CustomInfo":{"CustomField1":"v1.6","CustomField2":"14758f1afd44c09b7992073ccf00b43d","CustomField3":"my PC name","CustomField4":"personal","CustomField5":""},"Payload":"{\"10001\":\"my PC name\",\"10002\":\"personal\",\"10022\":\"00000000\",\"10029\":157,\"10006\":\"00\"}"}
diff --git a/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py b/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py
@@ -189,6 +189,46 @@
         'tlp': 'GREEN',
         'extra.source.connection_type': 'Cellular',
     },
+    {'__type': 'Event',
+     'classification.type': 'infected-system',
+     'destination.ip': '10.0.0.2',
+     'destination.port': 443,
+     'event_description.text': 'Microsoft.DCU.CTIP.Sinkhole',
+     'extra.custom_field1': 'v1.6',
+     'extra.custom_field2': '14758f1afd44c09b7992073ccf00b43d',
+     'extra.custom_field3': 'my PC name',
+     'extra.custom_field4': 'personal',
+     'extra.http.host': 'example.com',
+     'extra.http.method': 'POST',
+     'extra.http.request': '/index.php',
+     'extra.http.version': 'HTTP/1.1',
+     'extra.malware': 'Malware',
+     'extra.payload.10001': 'my PC name',
+     'extra.payload.10002': 'personal',
+     'extra.payload.10006': '00',
+     'extra.payload.10022': '00000000',
+     'extra.payload.10029': 157,
+     'extra.source.connection_type': 'Cable/DSL',
+     'extra.source.geolocation.postal_code': '66111',
+     'extra.total_encounters': 137,
+     'extra.user_agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) '
+     'AppleWebKit/537.36 (KHTML, like Gecko) '
+     'Chrome/79.0.3945.88 Safari/537.36',
+     'feed.accuracy': 100.0,
+     'feed.name': 'ctip',
+     'malware.name': 'b00-leet',
+     'raw': base64_encode(EXAMPLE_LINES[5]),
+     'source.as_name': 'My ISP',
+     'source.asn': 64496,
+     'source.geolocation.cc': 'DE',
+     'source.geolocation.city': 'Saarbrücken',
+     'source.geolocation.latitude': 49.2367,
+     'source.geolocation.longitude': 6.9794,
+     'source.geolocation.region': 'Saarland',
+     'source.ip': '10.0.0.15',
+     'source.port': 10000,
+     'time.source': '2022-06-06T16:59:01.802999+00:00',
+     'tlp': 'GREEN'},
 ]