Remove Malwaredomains parser and feed

The malwaredomains.com website is down, therefore we remove the
corresponding feed and parser. There were also some documentation
examples that mentioned that parser, that were changed accordingly.
The tests for the parser were removed and an upgrade function points out
existing stale malwaredomains parser configurations to the users.

Closes #2024

CHANGELOG.md

@@ -27,6 +27,7 @@ CHANGELOG
 
 #### Parsers
 - `intelmq.bots.parsers.microsoft.parser_ctip`: Map `Payload.domain` to `destination.fqdn` instead of `extra.payload.domain` as it matches to `destination.ip` from `DestinationIp` (PR#2023 by Sebastian Wagner).
+- Removed `intelmq.bots.parsers.malwaredomains` because the upstream data source (malwaredomains.com) does not exist anymore (PR#2026 by Birger Schacht, fixes #2024).
 
 #### Experts
 - `intelmq.bots.experts.truncate_by_delimiter.expert`: Cut string if its length is higher than a maximum length (PR#1967 by Marius Karotkis).
@@ -37,6 +38,7 @@ CHANGELOG
 
 ### Documentation
 - Various formatting fixes (by Sebastian Wagner).
+- Removed the malwaredomains feed from the feeds list because the upstream data source (malwaredomains.com) does not exist anymore (PR#2026 by Birger Schacht, fixes #2024).
 
 ### Packaging
 - intelmq-update-database crontab: Add missing `recordedfuture_iprisk` update call (by Sebastian Wagner).

NEWS.md

@@ -17,6 +17,9 @@ Please refer to the changelog for a full list of changes.
 
 ### Tools
 
+### Bots
+The malwardomains parser bot was removed. The malwaredomains.com website is offline, therefore the parser can not be used anymore. The `intelmqctl upgrade-config` command warns if you have the feed and the bot in use.
+
 ### Data Format
 
 ### Configuration

docs/user/configuration-management.rst

@@ -223,14 +223,14 @@ This configuration is used by each bot to load its specific (runtime) parameters
 
 .. code-block:: yaml
 
-   malware-domain-list-collector:
+   blocklistde-apache-collector:
      group: Collector
-     name: Malware Domain List
+     name: Blocklist.de Apache List
      module: intelmq.bots.collectors.http.collector_http
-     description: Malware Domain List Collector is the bot responsible to get the report from source of information.
+     description: Blocklist.de Apache Collector fetches all IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks.
      parameters:
-       http_url: http://www.malwaredomainlist.com/updatescsv.php
-       feed: Malware Domain List
+       http_url: https://lists.blocklist.de/lists/apache.txt
+       name: Blocklist.de Apache
        rate_limit: 3600
 
 More examples can be found in the ``intelmq/etc/runtime.conf`` directory. See :doc:`bots` for more details.
@@ -239,16 +239,16 @@ By default, all of the bots are started when you start the whole botnet, however
 
 .. code-block:: yaml
 
-    malware-domain-list-collector:
-      group: Collector
-      name: Malware Domain List
-      module: intelmq.bots.collectors.http.collector_http
-      description: Malware Domain List Collector is the bot responsible to get the report from source of information.
-      enabled: false,
-      parameters:
-        http_url: http://www.malwaredomainlist.com/updatescsv.php
-        feed: Malware Domain List
-        rate_limit: 3600
+   blocklistde-apache-collector:
+     group: Collector
+     name: Blocklist.de Apache List
+     module: intelmq.bots.collectors.http.collector_http
+     description: Blocklist.de Apache Collector fetches all IP addresses which have been reported within the last 48 hours as having run attacks on the service Apache, Apache-DDOS, RFI-Attacks.
+     enabled: false
+     parameters:
+       http_url: https://lists.blocklist.de/lists/apache.txt
+       name: Blocklist.de Apache
+       rate_limit: 3600
 
 Pipeline Configuration
 ======================

intelmq/etc/feeds.yaml

@@ -96,24 +96,6 @@ providers:
       revision: 2018-01-20
       documentation: https://www.malwarepatrol.net/non-commercial/
       public: false
-  Malware Domains:
-    Malicious:
-      description: Malware Prevention through Domain Blocking (Black Hole DNS Sinkhole)
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: http://mirror1.malwaredomains.com/files/domains.txt
-            rate_limit: 172800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malwaredomains.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://www.malwaredomains.com/
-      public: true
   ZoneH:
     Defacements:
       description: all the information contained in Zone-H's cybercrime archive were

intelmq/lib/upgrades.py

@@ -35,7 +35,8 @@
            'v233_feodotracker_browse',
            'v300_bots_file_removal',
            'v300_defaults_file_removal',
-           'v300_pipeline_file_removal'
+           'v300_pipeline_file_removal',
+           'v301_deprecations',
            ]
 
 
@@ -662,6 +663,32 @@ def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run):
     return messages if messages else changed, defaults, runtime, harmonization
 
 
+def v301_deprecations(defaults, runtime, harmonization, dry_run):
+    """
+    Deprecate malwaredomains parser and collector
+    """
+    found_malwaredomainsparser = []
+    found_malwaredomainscollector = []
+    changed = None
+    messages = []
+    for bot_id, bot in runtime.items():
+        if bot["module"] == "intelmq.bots.parsers.malwaredomains.parser":
+            found_malwaredomainsparser.append(bot_id)
+        if bot["module"] == "intelmq.bots.collectors.http.collector":
+            if "http_url" not in bot["parameters"]:
+                continue
+            if bot["parameters"]["http_url"] == 'http://mirror1.malwaredomains.com/files/domains.txt':
+                found_malwaredomainscollector.append(bot_id)
+    if found_malwaredomainsparser:
+        messages.append('A discontinued bot "Malware Domains Parser" has been found '
+                        'as bot %s.' % ', '.join(sorted(found_malwaredomainsparser)))
+    if found_malwaredomainscollector:
+        messages.append('A discontinued bot "Malware Domains Collector" has been found '
+                        'as bot %s.' % ', '.join(sorted(found_malwaredomainscollector)))
+    messages = ' '.join(messages)
+    return messages + ' Remove affected bots yourself.' if messages else changed, defaults, runtime, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -684,7 +711,7 @@ def v300_pipeline_file_removal(defaults, runtime, harmonization, dry_run):
     ((2, 3, 2), ()),
     ((2, 3, 3), (v233_feodotracker_browse, )),
     ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )),
-    ((3, 0, 1), ()),
+    ((3, 0, 1), (v301_deprecations, )),
 ])
 
 ALWAYS = (harmonization, )

intelmq/tests/lib/test_upgrades.py

@@ -489,7 +489,19 @@
     }
 }
 }
-
+V301_MALWAREDOMAINS_IN = {
+"malwaredomains-parser": {
+    "module": "intelmq.bots.parsers.malwaredomains.parser",
+    "parameters": {
+    }
+},
+"malwaredomains-collector": {
+    "module": "intelmq.bots.collectors.http.collector",
+    "parameters": {
+        "http_url": "http://mirror1.malwaredomains.com/files/domains.txt"
+        }
+    }
+}
 def generate_function(function):
     def test_function(self):
         """ Test if no errors happen for upgrade function %s. """ % function.__name__
@@ -677,6 +689,15 @@ def test_v233_feodotracker_browse(self):
         self.assertTrue(result[0])
         self.assertEqual(V233_FEODOTRACKER_BROWSE_OUT, result[2])
 
+    def test_v301_feed_changes(self):
+        """ Test v301_feed_changes """
+        result = upgrades.v301_deprecations({}, V301_MALWAREDOMAINS_IN, {}, False)
+        self.assertTrue(result[0])
+        self.assertEqual('A discontinued bot "Malware Domains Parser" has been found as bot '
+                         'malwaredomains-parser. A discontinued bot "Malware Domains Collector" '
+                         'has been found as bot malwaredomains-collector. Remove affected bots yourself.',
+                         result[0])
+        self.assertEqual(V301_MALWAREDOMAINS_IN, result[2])
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,