diff --git a/CHANGELOG.md b/CHANGELOG.md index a61a17826..e9831b042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ CHANGELOG - `intelmq.bots.parsers.microsoft.parser_ctip`: - New parameter `overwrite` (PR#2112 by Sebastian Wagner, fixes #2022). - Fix handling of field `Payload.domain` if it contains the same IP address as `Payload.serverIp` (PR#2144 by Mikk Margus Möll and Sebastian Wagner). + - Handle Payload field with non-base64-encoded JSON content and numbered dictionaries (PR#2193 by Sebastian Wagner) - `intelmq.bot.parsers.shodan.parser` (PR#2117 by Mikk Margus Möll): - Instead of keeping track of `extra.ftp..parameters`, FTP parameters are collected together into `extra.ftp.features` as a list of said features, reducing field count. - Shodan field `rsync.modules` is collected. diff --git a/intelmq/bots/parsers/microsoft/parser_ctip.py b/intelmq/bots/parsers/microsoft/parser_ctip.py index 6ca66d69b..bade0594a 100644 --- a/intelmq/bots/parsers/microsoft/parser_ctip.py +++ b/intelmq/bots/parsers/microsoft/parser_ctip.py @@ -59,7 +59,7 @@ "CustomField4": "", "CustomField5": "" }, - "Payload": base64 encoded json + "Payload": base64 encoded json with meaningful dictionary keys or JSON-string with numbered dictionary keys } """ @@ -263,14 +263,23 @@ def parse_azure(self, line, report): for key, value in line.copy().items(): if key == 'Payload': + # empty if value == 'AA==': # NULL del line[key] continue - try: - value = json.loads(utils.base64_decode(value)) - # continue unpacking in next loop - except json.decoder.JSONDecodeError: - line[key] = utils.base64_decode(value) + + # JSON string + if value.startswith('{'): + for payload_key, payload_value in json.loads(value).items(): + event[f'extra.payload.{payload_key}'] = payload_value + del line[key] + else: + # base64-encoded JSON + try: + value = json.loads(utils.base64_decode(value)) + # continue unpacking in next loop + except json.decoder.JSONDecodeError: + line[key] = utils.base64_decode(value) elif key == 'TLP' and value.lower() == 'unknown': del line[key] if isinstance(value, dict): diff --git a/intelmq/etc/feeds.yaml b/intelmq/etc/feeds.yaml index 4f0eb7f51..7de93404a 100644 --- a/intelmq/etc/feeds.yaml +++ b/intelmq/etc/feeds.yaml @@ -723,7 +723,7 @@ providers: services on the router or tried to gain access to them. The list also contains a list of tags for each address which indicate what behaviour of the address was observed. - + The Turris Greylist feed provides PGP signatures for the provided files. You will need to import the public PGP key from the linked documentation page, currently available at @@ -731,13 +731,13 @@ providers: or from below. See the URL Fetcher Collector documentation for more information on PGP signature verification. - + PGP Public key: ``` -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.1.6 Comment: Hostname: pgp.mit.edu - + mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0 o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t 3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40 @@ -1756,7 +1756,7 @@ providers: parser: module: intelmq.bots.parsers.microsoft.parser_ctip parameters: - revision: 2020-05-29 + revision: 2022-06-01 documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange http://www.dcuctip.com/ public: false CTIP C2 via Azure: @@ -1887,10 +1887,10 @@ providers: listen 443 ssl http2; server_name [your host name]; client_max_body_size 50M; - + ssl_certificate [path to your key]; ssl_certificate_key [path to your certificate]; - + location /[your private url] { if ($http_authorization != '[your private password]') { return 403; diff --git a/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt b/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt index 7a4dff4a2..03371ce50 100644 --- a/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt +++ b/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt @@ -3,3 +3,4 @@ {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"} {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"} {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiMTAuMC4wLjEiLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"} +{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132990083418030000,"DateTimeReceivedUtcTxt":"Wednesday June 01 2022 13:33:13.3713","Malware":"Malware","ThreatCode":"B00-Leet","ThreatConfidence":"High","TotalEncounters":137,"TLP":"Green","SourceIp":"10.0.0.15","SourcePort":10000,"DestinationIp":"10.0.0.2","DestinationPort":443,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"My ISP","SourceIpCountryCode":"DE","SourceIpRegion":"Saarland","SourceIpCity":"Saarbrücken","SourceIpPostalCode":"66111","SourceIpLatitude":49.2367,"SourceIpLongitude":6.9794,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cable/DSL","SourceIpv4Int":167772175},"HttpInfo":{"HttpHost":"example.com","HttpRequest":"/index.php","HttpMethod":"POST","HttpReferrer":"","HttpUserAgent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36","HttpVersion":"HTTP/1.1"},"CustomInfo":{"CustomField1":"v1.6","CustomField2":"14758f1afd44c09b7992073ccf00b43d","CustomField3":"my PC name","CustomField4":"personal","CustomField5":""},"Payload":"{\"10001\":\"my PC name\",\"10002\":\"personal\",\"10022\":\"00000000\",\"10029\":157,\"10006\":\"00\"}"} \ No newline at end of file diff --git a/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py b/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py index fe4ab9a22..a199a194e 100644 --- a/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py +++ b/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py @@ -189,6 +189,46 @@ 'tlp': 'GREEN', 'extra.source.connection_type': 'Cellular', }, + {'__type': 'Event', + 'classification.type': 'infected-system', + 'destination.ip': '10.0.0.2', + 'destination.port': 443, + 'event_description.text': 'Microsoft.DCU.CTIP.Sinkhole', + 'extra.custom_field1': 'v1.6', + 'extra.custom_field2': '14758f1afd44c09b7992073ccf00b43d', + 'extra.custom_field3': 'my PC name', + 'extra.custom_field4': 'personal', + 'extra.http.host': 'example.com', + 'extra.http.method': 'POST', + 'extra.http.request': '/index.php', + 'extra.http.version': 'HTTP/1.1', + 'extra.malware': 'Malware', + 'extra.payload.10001': 'my PC name', + 'extra.payload.10002': 'personal', + 'extra.payload.10006': '00', + 'extra.payload.10022': '00000000', + 'extra.payload.10029': 157, + 'extra.source.connection_type': 'Cable/DSL', + 'extra.source.geolocation.postal_code': '66111', + 'extra.total_encounters': 137, + 'extra.user_agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/79.0.3945.88 Safari/537.36', + 'feed.accuracy': 100.0, + 'feed.name': 'ctip', + 'malware.name': 'b00-leet', + 'raw': base64_encode(EXAMPLE_LINES[5]), + 'source.as_name': 'My ISP', + 'source.asn': 64496, + 'source.geolocation.cc': 'DE', + 'source.geolocation.city': 'Saarbrücken', + 'source.geolocation.latitude': 49.2367, + 'source.geolocation.longitude': 6.9794, + 'source.geolocation.region': 'Saarland', + 'source.ip': '10.0.0.15', + 'source.port': 10000, + 'time.source': '2022-06-06T16:59:01.802999+00:00', + 'tlp': 'GREEN'}, ]