Merge pull request #2227 from certtools/shadowserver-202207

Shadowserver 202207
certtools · Aug 3, 2022 · 572a36d · 572a36d
2 parents cfe14cb + 1e4a16c
commit 572a36d
Show file tree

Hide file tree

Showing 183 changed files with 7,812 additions and 4,937 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -9,11 +9,30 @@ NEWS
 This file lists all changes which have an affect on the administration of IntelMQ and contains steps that you need to be aware off for the upgrade.
 Please refer to the changelog for a full list of changes.
 
+
 3.1.0 Feature release (unreleased)
 ----------------------------------
 
 ### Requirements
 
+### Bots
+#### ShadowServer Reports API collector
+The misleading `country` parameter has been depreciated and a `reports` parameter has been added.
+The backwards-compatibility will be removed in IntelMQ version 4.0.0.
+See the [Shadowserver Reports API bot's documentation](https://intelmq.readthedocs.io/en/latest/user/bots.html#shadowserver-reports-api).
+
+
+3.1.0 Feature release (unreleased)
+----------------------------------
+
+### Requirements
+
+### Bots
+#### ShadowServer Reports API collector
+The misleading `country` parameter has been depreciated and a `reports` parameter has been added.
+The backwards-compatibility will be removed in IntelMQ version 4.0.0.
+See the [Shadowserver Reports API bot's documentation](https://intelmq.readthedocs.io/en/latest/user/bots.html#shadowserver-reports-api).
+
 ### Tools
 
 ### Data Format
@@ -37,12 +56,97 @@ The parameter `timeout` has been merged into `redis_cache_ttl`.
 ### Libraries
 
 ### Postgres databases
+The following statements optionally update existing data for the harmonization classification changes:
+```sql
+UPDATE events
+   SET "classification.identifier" = 'open-adb'
+   WHERE "classification.identifier" = 'accessible-adb';
+UPDATE events
+   SET "classification.identifier" = 'open-afp'
+   WHERE "classification.identifier" = 'accessible-afp';
+UPDATE events
+   SET "classification.identifier" = 'open-amqp'
+   WHERE "classification.identifier" = 'accessible-amqp';
+UPDATE events
+   SET "classification.identifier" = 'open-ard'
+   WHERE "classification.identifier" = 'accessible-ard';
+UPDATE events
+   SET "classification.identifier" = 'open-cisco-smart-install'
+   WHERE "classification.identifier" = 'accessible-cisco-smart-install';
+UPDATE events
+   SET "classification.identifier" = 'open-coap'
+   WHERE "classification.identifier" = 'accessible-coap';
+UPDATE events
+   SET "classification.identifier" = 'open-ftp'
+   WHERE "classification.identifier" = 'accessible-ftp';
+UPDATE events
+   SET "classification.identifier" = 'open-hadoop'
+   WHERE "classification.identifier" = 'accessible-hadoop';
+UPDATE events
+   SET "classification.identifier" = 'open-http'
+   WHERE "classification.identifier" = 'accessible-http';
+UPDATE events
+   SET "classification.identifier" = 'open-rdpeudp'
+   WHERE "classification.identifier" = 'accessible-msrdpeudp';
+UPDATE events
+   SET "classification.identifier" = 'open-radmin'
+   WHERE "classification.identifier" = 'accessible-radmin';
+UPDATE events
+   SET "classification.identifier" = 'open-rsync'
+   WHERE "classification.identifier" = 'accessible-rsync';
+UPDATE events
+   SET "classification.identifier" = 'open-ubiquiti'
+   WHERE "classification.identifier" = 'accessible-ubiquiti-discovery-service';
+UPDATE events
+   SET "classification.identifier" = 'honeypot-ddos-amp'
+   WHERE "classification.identifier" = 'amplification-ddos-victim';
+UPDATE events
+   SET "classification.identifier" = 'blocklist'
+   WHERE "classification.identifier" = 'blacklisted-ip';
+UPDATE events
+   SET "classification.identifier" = 'open-dns'
+   WHERE "classification.identifier" = 'dns-open-resolver';
+UPDATE events
+   SET "classification.identifier" = 'honeypot-http-scan'
+   WHERE "classification.identifier" = 'honeypot-http-scan';
+UPDATE events
+   SET "classification.identifier" = 'honeypot-ics-scan'
+   WHERE "classification.identifier" = 'ics';
+UPDATE events
+   SET "classification.identifier" = 'open-ntpmonitor'
+   WHERE "classification.identifier" = 'ntp-monitor';
+UPDATE events
+   SET "classification.identifier" = 'open-ntp'
+   WHERE "classification.identifier" = 'ntp-version';
+UPDATE events
+   SET "classification.identifier" = 'open-db2'
+   WHERE "classification.identifier" = 'open-db2-discovery-service';
+UPDATE events
+   SET "classification.identifier" = 'open-isakmp'
+   WHERE "classification.identifier" = 'open-ike';
+UPDATE events
+   SET "classification.identifier" = 'open-ldap-tcp'
+   WHERE "classification.identifier" = 'open-ldap';
+UPDATE events
+   SET "classification.identifier" = 'open-nat-pmp'
+   WHERE "classification.identifier" = 'open-natpmp';
+UPDATE events
+   SET "classification.identifier" = 'open-netbios'
+   WHERE "classification.identifier" = 'open-netbios-nameservice';
+UPDATE events
+   SET "classification.identifier" = 'open-netis-router'
+   WHERE "classification.identifier" = 'open-netis';
+UPDATE events
+   SET "classification.identifier" = 'sinkhole-dns'
+   WHERE "classification.identifier" = 'sinkholedns';
+```
 
 
 ### Bots
 
 #### Github Collector
 GitHub removed the basic `Username/Password` Authentication in favor of personal access tokens. So the GitHub Collector uses an Personal Access Token for authentication [Github Documentation: Generate a personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+=======
 
 
 3.0.2 Maintenance release (2021-09-10)

diff --git a/docs/user/bots.rst b/docs/user/bots.rst
@@ -654,9 +654,10 @@ The Cache is required to memorize which files have already been processed (TTL n
 
 **Configuration Parameters**
 
-* `country`: The country you want to download the reports for
+* `country`: **Deprecated:** The country you want to download the reports for. Will be removed in IntelMQ version 4.0.0, use *reports* instead.
 * `apikey`: Your Shadowserver API key
 * `secret`: Your Shadowserver API secret
+* `reports`: A list of strings or a comma-separated list of the mailing lists you want to process.
 * `types`: A list of strings or a string of comma-separated values with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section :ref:`Supported Reports <shadowserver-supported-reports>` of the Shadowserver parser.
 * **Cache parameters** (see in section :ref:`common-parameters`, the default TTL is set to 10 days)
 

diff --git a/intelmq/bots/collectors/shadowserver/README.md b/intelmq/bots/collectors/shadowserver/README.md
@@ -0,0 +1,8 @@
+<!--
+SPDX-FileCopyrightText: 2022 The Shadowserver Foundation
+SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+This module is maintained by [The Shadowserver Foundation](https://www.shadowserver.org/).  
+
+Please contact intelmq@shadowserver.org with any issues or concerns.
diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py
@@ -5,6 +5,7 @@
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
 from datetime import datetime, timedelta
+import json
 import hashlib
 import hmac
 import re
@@ -22,12 +23,14 @@
 
 class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
     """
-    Connects to the Shadowserver API, requests a list of all the reports for a specific country and processes the ones that are new
+    Connects to the Shadowserver API, requests a list of all the reports for an organization and processes the ones that are new
 
     Parameters:
         api_key (str): Your Shadowserver API key
         secret (str): Your Shadowserver API secret
-        country (str): The country you want to download reports for (i.e. 'austria')
+        country (str): DEPRECIATED The mailing list you want to download reports for (i.e. 'austria')
+        reports (list):
+            A list of strings or a comma-separated list of the mailing lists you want to process.
         types (list):
             A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined').
     """
@@ -36,23 +39,29 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin):
     api_key = None
     secret = None
     types = None
+    reports = None
     rate_limit: int = 86400
     redis_cache_db: int = 12
     redis_cache_host: str = "127.0.0.1"  # TODO: type could be ipadress
     redis_cache_port: int = 6379
     redis_cache_ttl: int = 864000  # 10 days
     redis_cache_password: Optional[str] = None
+    _report_list = []
 
     def init(self):
         if self.api_key is None:
             raise ValueError('No api_key provided.')
         if self.secret is None:
             raise ValueError('No secret provided.')
-        if self.country is None:
-            raise ValueError('No country provided.')
 
-        if isinstance(self.types, str):
-            self.types = self.types.split(',')
+        if isinstance(self.reports, str):
+            self._report_list = self.reports.split(',')
+        elif isinstance(self.reports, list):
+            self._report_list = self.reports
+
+        if self.country is not None and self.country not in self._report_list:
+            self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.")
+            self._report_list.append(self.country)
 
         self.preamble = f'{{ "apikey": "{self.api_key}" '
 
@@ -61,7 +70,7 @@ def _headers(self, data):
 
     def _reports_list(self, date=None):
         """
-        Get a list of all the reports shadowserver has for a specific country
+        Get a list of all the reports shadowserver has for an organization
         via the reports/list endpoint. If a list of types is set in the
         parameters, we only process reports with those types.
         To be on the safe side regarding different calculations of timestamps,
@@ -76,8 +85,9 @@ def _reports_list(self, date=None):
         dayafter = date + timedelta(1)
 
         data = self.preamble
-        data += f',"report": ["{self.country}"] '
-        data += f',"date": "{daybefore.isoformat()}:{dayafter.isoformat()}" '
+        data += ',"date": "{}:{}" '.format(daybefore.isoformat(), dayafter.isoformat())
+        if len(self._report_list) > 0:
+            data += ',"reports": {}'.format(json.dumps(self._report_list))
         data += '}'
         self.logger.debug('Downloading report list with data: %s.', data)
 

diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md
@@ -0,0 +1,9 @@
+<!--
+SPDX-FileCopyrightText: 2022 The Shadowserver Foundation
+SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+This module is maintained by [The Shadowserver Foundation](https://www.shadowserver.org/).  
+
+Please contact intelmq@shadowserver.org with any issues or concerns.
+