From 1eb14d6f7195d319237f23652dcabdacd29b9033 Mon Sep 17 00:00:00 2001 From: Sebastian Waldbauer Date: Thu, 14 Jul 2022 15:10:51 +0200 Subject: [PATCH] FIX: Remove AutoShun from IntelMQ as the feed is discontinued Fixes #2162 Signed-off-by: Sebastian Waldbauer --- CHANGELOG.md | 1 + intelmq/bots/parsers/autoshun/__init__.py | 0 intelmq/bots/parsers/autoshun/parser.py | 63 ----- intelmq/etc/feeds.yaml | 18 -- intelmq/lib/upgrades.py | 24 +- .../tests/bots/parsers/autoshun/__init__.py | 0 .../tests/bots/parsers/autoshun/shunlist.html | 17 -- .../bots/parsers/autoshun/test_parser.py | 65 ----- intelmq/tests/lib/test_upgrades.py | 234 ++++++++++-------- 9 files changed, 154 insertions(+), 268 deletions(-) delete mode 100644 intelmq/bots/parsers/autoshun/__init__.py delete mode 100644 intelmq/bots/parsers/autoshun/parser.py delete mode 100644 intelmq/tests/bots/parsers/autoshun/__init__.py delete mode 100644 intelmq/tests/bots/parsers/autoshun/shunlist.html delete mode 100644 intelmq/tests/bots/parsers/autoshun/test_parser.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 347d379cb..973eb970d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ CHANGELOG - Added file name mapping for `SSL-POODLE-Vulnerable-Servers IPv6` (file name `scan6_ssl_poodle`) (PR#2134 by Mateo Durante). - `intelmq.bots.parsers.cymru.parser_cap_program`: The parser mapped the hostname into `source.fqdn` which is not allowed by the IntelMQ Data Format. Added a check (PR#2215 by Sebastian Waldbauer, fixes #2169) - `intelmq.bots.parsers.generic.parser_csv`: Use RewindableFileHandle to use the original current line for line recovery (PR#2192 by Sebastian Wagner). +- `intelmq.bots.parsers.autoshun.parser`: Removed, as the feed is discontinued (PR#2214 by Sebastian Waldbauer, fixes #2162). #### Experts - `intelmq.bots.experts.domain_valid`: New bot for checking domain's validity (PR#1966 by Marius Karotkis). diff --git a/intelmq/bots/parsers/autoshun/__init__.py b/intelmq/bots/parsers/autoshun/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/intelmq/bots/parsers/autoshun/parser.py b/intelmq/bots/parsers/autoshun/parser.py deleted file mode 100644 index a8d60d3da..000000000 --- a/intelmq/bots/parsers/autoshun/parser.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-FileCopyrightText: 2015 National CyberSecurity Center -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import html - -from intelmq.lib import utils -from intelmq.lib.bot import ParserBot -from intelmq.lib.harmonization import ClassificationType - -TAXONOMY = { - "brute force": "brute-force", - "bruteforce": "brute-force", - "scan": "scanner", - "cve": "exploit", - "sql inject": "exploit", -} - - -class AutoshunParserBot(ParserBot): - """Parse the Autoshun feed""" - - def parse(self, report): - raw_report = utils.base64_decode(report.get("raw")) - split = raw_report.split("") - self.tempdata = [''.join(split[:2])] - # TODO: save ending line - for line in split[2:]: - yield line.strip() - - def parse_line(self, line, report): - event = self.new_event(report) - - info = line.split("") - if len(line) <= 0 or len(info) < 3: - return - - ip = info[1].split('')[0].strip() - last_seen = info[2].split('')[0].strip() + '-05:00' - description = html.unescape(info[3].split('')[0].strip()) - - for key in ClassificationType.allowed_values: - if description.lower().find(key.lower()) > -1: - event.add("classification.type", key) - break - else: - for key, value in TAXONOMY.items(): - if description.lower().find(key.lower()) > -1: - event.add("classification.type", value) - break - - if "classification.type" not in event: - event.add("classification.type", 'undetermined') - - event.add("time.source", last_seen) - event.add("source.ip", ip) - event.add("event_description.text", description) - event.add("raw", line + "") - yield event - - -BOT = AutoshunParserBot diff --git a/intelmq/etc/feeds.yaml b/intelmq/etc/feeds.yaml index 879704f7e..be9d6a2c6 100644 --- a/intelmq/etc/feeds.yaml +++ b/intelmq/etc/feeds.yaml @@ -856,24 +856,6 @@ providers: revision: 2018-01-20 documentation: https://dsi.ut-capitole.fr/blacklists/ public: true - Autoshun: - Shunlist: - description: You need to register in order to use the list. - additional_information: - bots: - collector: - module: intelmq.bots.collectors.http.collector_http - parameters: - http_url: https://www.autoshun.org/download/?api_key=__APIKEY__&format=html - rate_limit: 3600 - name: __FEED__ - provider: __PROVIDER__ - parser: - module: intelmq.bots.parsers.autoshun.parser - parameters: - revision: 2018-01-20 - documentation: https://www.autoshun.org/ - public: false Danger Rulez: Bruteforce Blocker: description: Its main purpose is to block SSH bruteforce attacks via firewall. diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 83dd28923..ca250df0c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -36,6 +36,7 @@ 'v300_defaults_file_removal', 'v300_pipeline_file_removal', 'v301_deprecations', + 'v310_feed_changes', ] @@ -725,6 +726,27 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs): return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization +def v310_feed_changes(configuration, harmonization, dry_run, **kwargs): + """ + Migrates feeds' configuration for changed/fixed parameter + """ + found_autoshun = [] + messages = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.collectors.http.collector": + if bot["parameters"].get("http_url", "").startswith("https://www.autoshun.org/download"): + found_autoshun.append(bot_id) + if bot["module"] == "intelmq.bots.parsers.autoshun.parser": + found_autoshun.append(bot_id) + if found_autoshun: + messages.append('A discontinued feed "Autoshun" has been found ' + f'as bot {", ".join(sorted(found_autoshun))}.') + messages = ' '.join(messages) + return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization + + UPGRADES = OrderedDict([ ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )), ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)), @@ -749,7 +771,7 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs): ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )), ((3, 0, 1), (v301_deprecations, )), ((3, 0, 2), ()), - ((3, 1, 0), ()), + ((3, 1, 0), (v310_feed_changes, )), ]) ALWAYS = (harmonization, ) diff --git a/intelmq/tests/bots/parsers/autoshun/__init__.py b/intelmq/tests/bots/parsers/autoshun/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/intelmq/tests/bots/parsers/autoshun/shunlist.html b/intelmq/tests/bots/parsers/autoshun/shunlist.html deleted file mode 100644 index c96e3f6d6..000000000 --- a/intelmq/tests/bots/parsers/autoshun/shunlist.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - -
Currently Shunning 500 attackers as of Tue, 31 May 2016 06:30:01 -0500
AttackerTime of Shun CDTDescription of Event
198.51.100.452016-05-17 10:59:27RA SCAN Unusually fast Terminal Server Traffic Inbound
198.51.100.862016-05-18 14:01:06MS Terminal Server Single Character Login, possible Morto inbound
diff --git a/intelmq/tests/bots/parsers/autoshun/test_parser.py b/intelmq/tests/bots/parsers/autoshun/test_parser.py deleted file mode 100644 index 11b54cf9e..000000000 --- a/intelmq/tests/bots/parsers/autoshun/test_parser.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: 2015 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.autoshun.parser import AutoshunParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'shunlist.html')) as handle: - EXAMPLE_FILE = handle.read() - -EXAMPLE_REPORT = {"feed.name": "Autoshun", - "feed.url": "https://www.autoshun.org/files/shunlist.html", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-09-02T14:17:58+00:00" - } -EXAMPLE_EVENT0 = { - "__type": "Event", - "feed.name": "Autoshun", - "classification.type": "scanner", - "feed.url": "https://www.autoshun.org/files/shunlist.html", - "raw": "PHRyPjx0ZD4xOTguNTEuMTAwLjQ1PC90ZD48dGQ+MjAxNi0wNS0xNyAxMDo1OToyNzwvdGQ+PHRkPlJBIFNDQU4gVW51c3VhbGx5IGZhc3QgVGVybWluYWwgU2VydmVyIFRyYWZmaWMgSW5ib3VuZDwvdGQ+PC90cj4=", - "event_description.text": "RA SCAN Unusually fast Terminal Server Traffic Inbound", - "time.source": "2016-05-17T15:59:27+00:00", - "time.observation": "2015-05-17T14:17:5810:59:27+00:00", - "source.ip": "198.51.100.45" -} -EXAMPLE_EVENT1 = { - "__type": "Event", - "feed.name": "Autoshun", - "classification.type": "undetermined", - "feed.url": "https://www.autoshun.org/files/shunlist.html", - "raw": "PHRyPjx0ZD4xOTguNTEuMTAwLjg2PC90ZD48dGQ+MjAxNi0wNS0xOCAxNDowMTowNjwvdGQ+PHRkPk1TIFRlcm1pbmFsIFNlcnZlciBTaW5nbGUgQ2hhcmFjdGVyIExvZ2luLCBwb3NzaWJsZSBNb3J0byBpbmJvdW5kPC90ZD48L3RyPg==", - "event_description.text": "MS Terminal Server Single Character Login, possible Morto inbound", - "time.source": "2016-05-18T19:01:06+00:00", - "time.observation": "2015-05-17T14:17:5810:59:27+00:00", - "source.ip": "198.51.100.86" -} - - -class TestAutoshunParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for AutoshunParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = AutoshunParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - self.assertMessageEqual(0, EXAMPLE_EVENT0) - self.assertMessageEqual(1, EXAMPLE_EVENT1) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 8a6e13b21..0c310c029 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -14,7 +14,7 @@ V202 = {"global": {}, -"test-collector": { + "test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { @@ -36,7 +36,7 @@ }, } V202_EXP = {"global": {}, -"test-collector": { + "test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { @@ -61,7 +61,7 @@ } DEP_110 = {"global": {}, -"n6-collector": { + "n6-collector": { "group": "Collector", "module": "intelmq.bots.collectors.n6.collector_stomp", "parameters": { @@ -83,7 +83,7 @@ } } DEP_110_EXP = {"global": {}, -"n6-collector": { + "n6-collector": { "group": "Collector", "module": "intelmq.bots.collectors.stomp.collector", "parameters": { @@ -104,7 +104,7 @@ }, }} V210 = {"global": {}, -"test-collector": { + "test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.rt.collector_rt", "parameters": { @@ -154,7 +154,7 @@ } } V210_EXP = {"global": {}, -"test-collector": { + "test-collector": { "group": "Collector", "module": "intelmq.bots.collectors.rt.collector_rt", "parameters": { @@ -206,7 +206,7 @@ } } V213 = {"global": {}, -"mail-collector": { + "mail-collector": { "group": "Collector", "module": "intelmq.bots.collectors.mail.collector_mail_attach", "parameters": { @@ -223,7 +223,7 @@ } } V213_EXP = {"global": {}, -"mail-collector": { + "mail-collector": { "group": "Collector", "module": "intelmq.bots.collectors.mail.collector_mail_attach", "parameters": { @@ -239,28 +239,28 @@ } } V220_MISP_VERIFY_FALSE = { -"global": {"http_verify_cert": True}, -"misp-collector": { + "global": {"http_verify_cert": True}, + "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { - "misp_verify": False}}} + "misp_verify": False}}} V220_MISP_VERIFY_NULL = { -"global": {"http_verify_cert": True}, -"misp-collector": { + "global": {"http_verify_cert": True}, + "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": {}}} V220_MISP_VERIFY_TRUE = { -"global": {"http_verify_cert": True}, -"misp-collector": { + "global": {"http_verify_cert": True}, + "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { - "misp_verify": True}}} + "misp_verify": True}}} V220_HTTP_VERIFY_FALSE = { -"global": {"http_verify_cert": True}, -"misp-collector": { + "global": {"http_verify_cert": True}, + "misp-collector": { "module": "intelmq.bots.collectors.misp.collector", "parameters": { - "http_verify_cert": False}}} + "http_verify_cert": False}}} HARM = load_configuration(pkg_resources.resource_filename('intelmq', 'etc/harmonization.conf')) V210_HARM = deepcopy(HARM) @@ -272,96 +272,96 @@ WRONG_REGEX = deepcopy(HARM) WRONG_REGEX['event']['protocol.transport']['iregex'] = 'foobar' V213_FEED = {"global": {}, -"zeus-collector": { + "zeus-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "https://zeustracker.abuse.ch/blocklist.php?download=badips", } }, -"bitcash-collector": { + "bitcash-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "https://bitcash.cz/misc/log/blacklist", } }, -"ddos-attack-c2-collector": { + "ddos-attack-c2-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http_stream", "parameters": { "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", } }, -"ddos-attack-targets-collector": { + "ddos-attack-targets-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http_stream", "parameters": { "http_url": "https://feed.caad.fkie.fraunhofer.de/ddosattackfeed/", } }, -"taichung-collector": { + "taichung-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "https://www.tc.edu.tw/net/netflow/lkout/recent/30", }, }, -"ransomware-collector": { + "ransomware-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "https://ransomwaretracker.abuse.ch/feeds/csv/", }, }, -"bambenek-dga-collector": { + "bambenek-dga-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "https://osint.bambenekconsulting.com/feeds/dga-feed.txt", }, }, -"bambenek-c2dommasterlist-collector": { + "bambenek-c2dommasterlist-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt", }, }, -"nothink-dns-collector": { + "nothink-dns-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "http://www.nothink.org/honeypot_dns_attacks.txt", }, }, -"nothink-ssh-collector": { + "nothink-ssh-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "http://www.nothink.org/blacklist/blacklist_ssh_day.txt", }, }, -"nothink-parser": { + "nothink-parser": { "group": "Parser", "module": "intelmq.bots.parsers.nothink.parser", }, } V220_FEED = {"global": {}, -"urlvir-hosts-collector": { + "urlvir-hosts-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "http://www.urlvir.com/export-hosts/", }, }, -"urlvir-parser": { + "urlvir-parser": { "group": "Parser", "module": "intelmq.bots.parsers.urlvir.parser", }, } V221_FEED = {"global": {}, -"abusech-urlhaus-columns-string-parser": { + "abusech-urlhaus-columns-string-parser": { "parameters": { "column_regex_search": {}, "columns": "time.source,source.url,status,extra.urlhaus.threat_type,source.fqdn,source.ip,source.asn,source.geolocation.cc", @@ -378,10 +378,10 @@ }, "module": "intelmq.bots.parsers.generic.parser_csv", }, -"abusech-urlhaus-columns-dict-parser": { + "abusech-urlhaus-columns-dict-parser": { "parameters": { "column_regex_search": {}, - "columns": ["time.source", "source.url","status","extra.urlhaus.threat_type","source.fqdn","source.ip","source.asn","source.geolocation.cc"], + "columns": ["time.source", "source.url", "status", "extra.urlhaus.threat_type", "source.fqdn", "source.ip", "source.asn", "source.geolocation.cc"], "default_url_protocol": "http://", "delimiter": ",", "filter_text": None, @@ -397,7 +397,7 @@ } } V221_FEED_OUT = {"global": {}, -"abusech-urlhaus-columns-string-parser": { + "abusech-urlhaus-columns-string-parser": { "parameters": { "column_regex_search": {}, "columns": ['time.source', 'source.url', 'status', 'classification.type|__IGNORE__', 'source.fqdn|__IGNORE__', 'source.ip', 'source.asn', 'source.geolocation.cc'], @@ -417,111 +417,127 @@ } V221_FEED_OUT['abusech-urlhaus-columns-dict-parser'] = V221_FEED_OUT['abusech-urlhaus-columns-string-parser'] V221_FEED_2 = {"global": {}, -"hphosts-collector": { + "hphosts-collector": { "group": "Collector", "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "http_url": "http://hosts-file.net/download/hosts.txt", }, }, -"hphosts-parser": { + "hphosts-parser": { "group": "Parser", "module": "intelmq.bots.parsers.hphosts.parser", }, } V222 = { -"global": {}, -"shadowserver-parser": { - "module": "intelmq.bots.parsers.shadowserver.parser", - "parameters": { - "feedname": "Blacklisted-IP"}}} + "global": {}, + "shadowserver-parser": { + "module": "intelmq.bots.parsers.shadowserver.parser", + "parameters": { + "feedname": "Blacklisted-IP"}}} V222_OUT = { -"global": {}, -"shadowserver-parser": { - "module": "intelmq.bots.parsers.shadowserver.parser", - "parameters": { - "feedname": "Blocklist"}}} + "global": {}, + "shadowserver-parser": { + "module": "intelmq.bots.parsers.shadowserver.parser", + "parameters": { + "feedname": "Blocklist"}}} V230_IN = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimeter": "," + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimeter": "," + } } } -} V230_IN_BOTH = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimeter": ",", - "delimiter": "," + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimeter": ",", + "delimiter": "," + } } } -} V230_OUT = { -"global": {}, -"urlhaus-parser": { - "module": "intelmq.bots.parsers.generic.parser_csv", - "parameters": { - "delimiter": "," + "global": {}, + "urlhaus-parser": { + "module": "intelmq.bots.parsers.generic.parser_csv", + "parameters": { + "delimiter": "," + } } } -} V230_MALWAREDOMAINLIST_IN = { -"global": {}, -"malwaredomainlist-parser": { - "module": "intelmq.bots.parsers.malwaredomainlist.parser", - "parameters": { - } -}, -"malwaredomainlist-collector": { - "module": "intelmq.bots.collectors.http.collector_http", - "parameters": { - "http_url": "http://www.malwaredomainlist.com/updatescsv.php" + "global": {}, + "malwaredomainlist-parser": { + "module": "intelmq.bots.parsers.malwaredomainlist.parser", + "parameters": { + } + }, + "malwaredomainlist-collector": { + "module": "intelmq.bots.collectors.http.collector_http", + "parameters": { + "http_url": "http://www.malwaredomainlist.com/updatescsv.php" } } } V233_FEODOTRACKER_BROWSE_IN = { -"global": {}, -'Feodo-tracker-browse-parser': { - 'module': "intelmq.bots.parsers.html_table.parser", - 'parameters': { - 'columns': 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc'.split(','), - 'type': 'c2server', - 'ignore_values': ',,,,Not listed,,', - 'skip_table_head': True, + "global": {}, + 'Feodo-tracker-browse-parser': { + 'module': "intelmq.bots.parsers.html_table.parser", + 'parameters': { + 'columns': 'time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc'.split(','), + 'type': 'c2server', + 'ignore_values': ',,,,Not listed,,', + 'skip_table_head': True, + } } } -} V233_FEODOTRACKER_BROWSE_OUT = { -"global": {}, -'Feodo-tracker-browse-parser': { - 'module': "intelmq.bots.parsers.html_table.parser", - 'parameters': { - 'columns': 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc', - 'type': 'c2server', - 'ignore_values': ',,,,,', - 'skip_table_head': True, + "global": {}, + 'Feodo-tracker-browse-parser': { + 'module': "intelmq.bots.parsers.html_table.parser", + 'parameters': { + 'columns': 'time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc', + 'type': 'c2server', + 'ignore_values': ',,,,,', + 'skip_table_head': True, + } } } -} V301_MALWAREDOMAINS_IN = { -"global": {}, -"malwaredomains-parser": { - "module": "intelmq.bots.parsers.malwaredomains.parser", - "parameters": { + "global": {}, + "malwaredomains-parser": { + "module": "intelmq.bots.parsers.malwaredomains.parser", + "parameters": { + } + }, + "malwaredomains-collector": { + "module": "intelmq.bots.collectors.http.collector", + "parameters": { + "http_url": "http://mirror1.malwaredomains.com/files/domains.txt" + } } -}, -"malwaredomains-collector": { - "module": "intelmq.bots.collectors.http.collector", - "parameters": { - "http_url": "http://mirror1.malwaredomains.com/files/domains.txt" +} +V310_FEED_CHANGES = { + "global": {}, + "autoshun-parser": { + "module": "intelmq.bots.parsers.autoshun.parser", + "parameters": { + } + }, + "autoshun-collector": { + "module": "intelmq.bots.collectors.http.collector", + "parameters": { + "http_url": "https://www.autoshun.org/download" } } } + + def generate_function(function): def test_function(self): """ Test if no errors happen for upgrade function %s. """ % function.__name__ @@ -718,6 +734,16 @@ def test_v301_feed_changes(self): result[0]) self.assertEqual(V301_MALWAREDOMAINS_IN, result[1]) + def test_v310_feed_changes(self): + """ Test v310_feed_changes """ + result = upgrades.v310_feed_changes(V310_FEED_CHANGES, {}, False) + self.assertTrue(result[0]) + self.assertEqual('A discontinued feed "Autoshun" has been found as bot ' + 'autoshun-collector, autoshun-parser. Remove affected bots yourself.', + result[0]) + self.assertEqual(V310_FEED_CHANGES, result[1]) + + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name, generate_function(getattr(upgrades, name)))