Implement a clear class hierarchy

CHANGELOG.md

@@ -17,12 +17,18 @@ CHANGELOG
   - Pass version history as parameter to upgrade functions (PR#2058 by Sebastian Wagner).
 - `intelmq.lib.message`:
   - Fix and pre-compile the regular expression for harmonization key names and also check keys in the `extra.` namespace (PR#2059 by Sebastian Wagner, fixes #1807).
+- `intelmq.lib.bot.SQLBot` was replaced by an SQLMixin in `intelmq.lib.mixins.SQLMixin`. The Generic DB Lookup Expert bot and the SQLOutput bot were updated accordingly.
+- Added an ExpertBot class - it should be used by all expert bots as a parent class
+- Introduced a module for IntelMQ related datatypes `intelmq.lib.datatypes` which for now only contains an Enum listing the four bot types
+- Added a `bottype` attribute to CollectorBot, ParserBot, ExpertBot, OutputBot
 
 ### Development
 
 ### Data Format
 
 ### Bots
+- Set the parent class of all bots to the correct bot class
+
 #### Collectors
 - `intelmq.bots.collectors.mail._lib`: Add support for unverified SSL/STARTTLS connections (PR#2055 by Sebastian Wagner).
 

docs/dev/guide.rst

@@ -451,6 +451,8 @@ Bot Developer Guide
 
 There's a dummy bot including tests at `intelmq/tests/lib/test_parser_bot.py`.
 
+Please use the correct bot type as parent class for your bot. The `intelmq.lib.bot` module contains the classes `CollectorBot`, `ParserBot`, `ExpertBot` and `OutputBot`.
+
 You can always start any bot directly from command line by calling the executable.
 The executable will be created during installation a directory for binaries. After adding new bots to the code, install IntelMQ to get the files created.
 Don't forget to give an bot id as first argument. Also, running bots with other users than `intelmq` will raise permission errors.
@@ -482,10 +484,10 @@ Please adjust the doc strings accordingly and remove the in-line comments (`#`).
    import sys
 
    # imports for additional libraries and intelmq
-   from intelmq.lib.bot import Bot
+   from intelmq.lib.bot import ParserBot
 
 
-   class ExampleParserBot(Bot):
+   class ExampleParserBot(ParserBot):
 
        option1: str = "defaultvalue"
        option2: bool = False
@@ -531,13 +533,40 @@ For common settings and methods you can use mixins from :code:`intelmq.lib.mixin
 
    class HTTPCollectorBot(CollectorBot, HttpMixin):
 
-At the moment there is a HttpMixin that you can use for common http requests.
-It provides the HTTP attributes described in :ref:`common-parameters` and the following methods:
+The following mixins are available:
+
+* `HttpMixin`
+* `SqlMixin`
+* `CacheMixin`
+
+The `HttpMixin` provides the HTTP attributes described in :ref:`common-parameters` and the following methods:
 
 * :code:`http_get` takes an URL as argument. Any other arguments get passed to the :code:`request.Session.get` method. :code:`http_get` returns a :code:`reqests.Response`.
 
 * :code:`http_session` can be used if you ever want to work with the session object directly. It takes no arguments and returns the bots request.Session.
 
+The `SqlMixin` provides methods to connect to SQL servers. Inherit this Mixin so that it handles DB connection for you.
+You do not have to bother:
+
+* connecting database in the :code:`self.init()` method, self.cur will be set in the :code:`__init__()`
+* catching exceptions, just call :code:`self.execute()` instead of :code:`self.cur.execute()`
+* :code:`self.format_char` will be set to '%s' in PostgreSQL and to '?' in SQLite
+
+The `CacheMixin` provides methods to cache values for bots in a Redis database. It uses the following attributes:
+
+* :code:`redis_cache_host: str = "127.0.0.1"`
+* :code:`redis_cache_port: int = 6379`
+* :code:`redis_cache_db: int = 9`
+* :code:`redis_cache_ttl: int = 15`
+* :code:`redis_cache_password: Optional[str] = None`
+
+and provides the methods:
+
+* :code:`cache_exists`
+* :code:`cache_get`
+* :code:`cache_set`
+* :code:`cache_flush`
+* :code:`cache_get_redis_instance`
 
 Pipeline interactions
 =====================

intelmq/bots/experts/abusix/expert.py

@@ -8,7 +8,7 @@
 RIPE abuse contacts resolving through DNS TXT queries
 '''
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 from ._lib import Abusix
 
@@ -18,7 +18,7 @@
     querycontacts = None
 
 
-class AbusixExpertBot(Bot):
+class AbusixExpertBot(ExpertBot):
     """Add abuse contact information from the Abusix online service for source and destination IP address"""
 
     def init(self):

intelmq/bots/experts/aggregate/expert.py

@@ -8,12 +8,12 @@
 from datetime import datetime, timedelta
 import time
 import json
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import parse_relative
 from intelmq.lib.mixins import CacheMixin
 
 
-class AggregateExpertBot(Bot, CacheMixin):
+class AggregateExpertBot(ExpertBot, CacheMixin):
     """Aggregation expert bot"""
 
     fields: str = "classification.type, classification.identifier"

intelmq/bots/experts/asn_lookup/expert.py

@@ -11,7 +11,7 @@
 import pathlib
 import requests
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -23,7 +23,7 @@
     pyasn = None
 
 
-class ASNLookupExpertBot(Bot):
+class ASNLookupExpertBot(ExpertBot):
     """Add ASN and netmask information from a local BGP dump"""
     database = None  # TODO: should be pathlib.Path
 

intelmq/bots/experts/csv_converter/expert.py

@@ -5,10 +5,10 @@
 # -*- coding: utf-8 -*-
 import csv
 import io
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class CSVConverterExpertBot(Bot):
+class CSVConverterExpertBot(ExpertBot):
     """Convert data to CSV"""
     fieldnames: str = "time.source,classification.type,source.ip"  # TODO: could maybe be List[str]
     delimiter: str = ','

intelmq/bots/experts/cymru_whois/expert.py

@@ -5,7 +5,7 @@
 # -*- coding: utf-8 -*-
 import json
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.harmonization import IPAddress
 from intelmq.lib.mixins import CacheMixin
 
@@ -14,7 +14,7 @@
 CACHE_KEY = "%d_%s"
 
 
-class CymruExpertBot(Bot, CacheMixin):
+class CymruExpertBot(ExpertBot, CacheMixin):
     """Add ASN, netmask, AS name, country, registry and allocation time from the Cymru Whois DNS service"""
     overwrite = False
     redis_cache_db: int = 5

intelmq/bots/experts/deduplicator/expert.py

@@ -26,11 +26,11 @@
                  system because system will always ignore this key.
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.mixins import CacheMixin
 
 
-class DeduplicatorExpertBot(Bot, CacheMixin):
+class DeduplicatorExpertBot(ExpertBot, CacheMixin):
     """Detection and drop exact duplicate messages. Message hashes are cached in the Redis database"""
     filter_keys: str = "raw,time.observation"  # TODO: could be List[str]
     filter_type: str = "blacklist"

intelmq/bots/experts/do_portal/expert.py

@@ -14,10 +14,10 @@
     requests = None
 
 import intelmq.lib.utils as utils
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class DoPortalExpertBot(Bot):
+class DoPortalExpertBot(ExpertBot):
     """Retrieve abuse contact information for the source IP address from a do-portal instance"""
     mode: str = "append"
     portal_api_key: str = None

intelmq/bots/experts/domain_suffix/expert.py

@@ -14,7 +14,7 @@
 
 import requests.exceptions
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import InvalidArgument
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -28,7 +28,7 @@
 ALLOWED_FIELDS = ['fqdn', 'reverse_dns']
 
 
-class DomainSuffixExpertBot(Bot):
+class DomainSuffixExpertBot(ExpertBot):
     """Extract the domain suffix from a domain and save it in the the domain_suffix field. Requires a local file with valid domain suffixes"""
     field: str = None
     suffix_file: str = None  # TODO: should be pathlib.Path

intelmq/bots/experts/domain_valid/expert.py

@@ -17,13 +17,13 @@
 
 import requests.exceptions
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError, ConfigurationError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
 
 
-class DomainValidExpertBot(Bot):
+class DomainValidExpertBot(ExpertBot):
     domain_field: str = 'source.fqdn'
     tlds_domains_list: str = '/opt/intelmq/var/lib/bots/domain_valid/tlds-alpha-by-domain.txt'
 

intelmq/bots/experts/field_reducer/expert.py

@@ -7,11 +7,11 @@
 Reducer bot
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.message import Event
 
 
-class FieldReducerExpertBot(Bot):
+class FieldReducerExpertBot(ExpertBot):
     """Remove fields from events"""
     type = None
     keys = None

intelmq/bots/experts/filter/expert.py

@@ -10,11 +10,11 @@
 import pytz
 from dateutil import parser
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import parse_relative, TIMESPANS
 
 
-class FilterExpertBot(Bot):
+class FilterExpertBot(ExpertBot):
     """Filter events, supports named paths for splitting the message flow"""
 
     _message_processed_verb = 'Forwarded'

intelmq/bots/experts/format_field/expert.py

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 # -*- coding: utf-8 -*-
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class FormatFieldExpertBot(Bot):
+class FormatFieldExpertBot(ExpertBot):
     """Perform string method operations on column values"""
     new_value       = ""
     old_value       = ""

intelmq/bots/experts/generic_db_lookup/expert.py

@@ -7,10 +7,11 @@
 Generic DB Lookup
 """
 
-from intelmq.lib.bot import SQLBot
+from intelmq.lib.bot import ExpertBot
+from intelmq.lib.mixins import SQLMixin
 
 
-class GenericDBLookupExpertBot(SQLBot):
+class GenericDBLookupExpertBot(ExpertBot, SQLMixin):
     """Fetche data from a database"""
     database: str = "intelmq"
     engine: str = "<postgresql OR sqlite>"

intelmq/bots/experts/geohash/expert.py

@@ -8,15 +8,15 @@
 https://pypi.org/project/geolib/
 https://github.com/joyanujoy/geolib
 '''
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 try:
     from geolib import geohash
 except ImportError:
     geohash = None
 
 
-class GeohashExpertBot(Bot):
+class GeohashExpertBot(ExpertBot):
     """Compute the geohash from longitude/latitude information, save it to extra.(source|destination)"""
     overwrite: bool = False
     precision: int = 7

intelmq/bots/experts/gethostbyname/expert.py

@@ -24,12 +24,12 @@
 """
 import socket
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.harmonization import URL
 from intelmq.lib.exceptions import InvalidArgument
 
 
-class GethostbynameExpertBot(Bot):
+class GethostbynameExpertBot(ExpertBot):
     """Resolve the IP address for the FQDN"""
     fallback_to_url: bool = True
     gaierrors_to_ignore = ()

intelmq/bots/experts/http/expert_content.py

@@ -6,11 +6,11 @@
 """
 from typing import List
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import create_request_session
 
 
-class HttpContentExpertBot(Bot):
+class HttpContentExpertBot(ExpertBot):
     """
     Test if a given string is part of the content for a given URL
 

intelmq/bots/experts/http/expert_status.py

@@ -6,11 +6,11 @@
 """
 from typing import List
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import create_request_session
 
 
-class HttpStatusExpertBot(Bot):
+class HttpStatusExpertBot(ExpertBot):
     """
     Fetch the HTTP Status for a given URL
 

intelmq/bots/experts/idea/expert.py

@@ -11,7 +11,7 @@
 from urllib.parse import quote_plus
 from uuid import uuid4
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
 def quot(s):
@@ -26,7 +26,7 @@ def addr6(s):
     return s if ":" in s else None
 
 
-class IdeaExpertBot(Bot):
+class IdeaExpertBot(ExpertBot):
     """Convert events into the IDEA format"""
     test_mode: bool = False
 

intelmq/bots/experts/lookyloo/expert.py

@@ -4,7 +4,7 @@
 
 # -*- coding: utf-8 -*-
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 
 try:
@@ -13,7 +13,7 @@
     pylookyloo = None
 
 
-class LookyLooExpertBot(Bot):
+class LookyLooExpertBot(ExpertBot):
     """ LookyLoo expert bot for automated website screenshots """
     instance_url: str = "http://localhost:5100/"
 

intelmq/bots/experts/maxmind_geoip/expert.py

@@ -14,7 +14,7 @@
 import requests
 import tarfile
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -25,7 +25,7 @@
     geoip2 = None
 
 
-class GeoIPExpertBot(Bot):
+class GeoIPExpertBot(ExpertBot):
     """Add geolocation information from a local MaxMind database to events (country, city, longitude, latitude)"""
     database: str = "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb"  # TODO: should be pathlib.Path
     license_key: str = "<insert Maxmind license key>"