FIX: Remove AutoShun from IntelMQ as the feed is discontinued

Fixes #2162

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

CHANGELOG.md

@@ -77,6 +77,7 @@ CHANGELOG
   - Added file name mapping for `SSL-POODLE-Vulnerable-Servers IPv6` (file name `scan6_ssl_poodle`) (PR#2134 by Mateo Durante).
 - `intelmq.bots.parsers.cymru.parser_cap_program`: The parser mapped the hostname into `source.fqdn` which is not allowed by the IntelMQ Data Format. Added a check (PR#2215 by Sebastian Waldbauer, fixes #2169)
 - `intelmq.bots.parsers.generic.parser_csv`: Use RewindableFileHandle to use the original current line for line recovery (PR#2192 by Sebastian Wagner).
+- `intelmq.bots.parsers.autoshun.parser`: Removed, as the feed is discontinued (PR#2214 by Sebastian Waldbauer, fixes #2162).
 
 #### Experts
 - `intelmq.bots.experts.domain_valid`: New bot for checking domain's validity (PR#1966 by Marius Karotkis).

intelmq/etc/feeds.yaml

@@ -856,24 +856,6 @@ providers:
       revision: 2018-01-20
       documentation: https://dsi.ut-capitole.fr/blacklists/
       public: true
-  Autoshun:
-    Shunlist:
-      description: You need to register in order to use the list.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://www.autoshun.org/download/?api_key=__APIKEY__&format=html
-            rate_limit: 3600
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.autoshun.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: https://www.autoshun.org/
-      public: false
   Danger Rulez:
     Bruteforce Blocker:
       description: Its main purpose is to block SSH bruteforce attacks via firewall.

intelmq/lib/upgrades.py

@@ -36,6 +36,7 @@
            'v300_defaults_file_removal',
            'v300_pipeline_file_removal',
            'v301_deprecations',
+           'v310_feed_changes',
            ]
 
 
@@ -725,6 +726,27 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs):
     return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization
 
 
+def v310_feed_changes(configuration, harmonization, dry_run, **kwargs):
+    """
+    Migrates feeds' configuration for changed/fixed parameter
+    """
+    found_autoshun = []
+    messages = []
+    for bot_id, bot in configuration.items():
+        if bot_id == 'global':
+            continue
+        if bot["module"] == "intelmq.bots.collectors.http.collector":
+            if bot["parameters"].get("http_url", "").startswith("https://www.autoshun.org/download"):
+                found_autoshun.append(bot_id)
+        if bot["module"] == "intelmq.bots.parsers.autoshun.parser":
+            found_autoshun.append(bot_id)
+    if found_autoshun:
+        messages.append('A discontinued feed "Autoshun" has been found '
+                        f'as bot {", ".join(sorted(found_autoshun))}.')
+    messages = ' '.join(messages)
+    return messages + ' Remove affected bots yourself.' if messages else None, configuration, harmonization
+
+
 UPGRADES = OrderedDict([
     ((1, 0, 0, 'dev7'), (v100_dev7_modify_syntax, )),
     ((1, 1, 0), (v110_shadowserver_feednames, v110_deprecations)),
@@ -749,7 +771,7 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs):
     ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )),
     ((3, 0, 1), (v301_deprecations, )),
     ((3, 0, 2), ()),
-    ((3, 1, 0), ()),
+    ((3, 1, 0), (v310_feed_changes, )),
 ])
 
 ALWAYS = (harmonization, )