Implement a clear class hierarchy

This commit introduces an ExpertBot class that all expert bots should
inherit from. To have a central place for storing the four types of bots
there is now a datatypes module that contains an enum listing the four
bot types. It can and should be used for more types that are part of the
IntelMQ codebase.
Also some output and parser bots were updated to depend on the correct
bot type.

CHANGELOG.md

@@ -13,12 +13,17 @@ CHANGELOG
 
 ### Core
 - `intelmq.lib.bot.SQLBot` was replaced by an SQLMixin in `intelmq.lib.mixins.SQLMixin`. The Generic DB Lookup Expert bot and the SQLOutput bot were updated accordingly.
+- Added an ExpertBot class - it should be used by all expert bots as a parent class
+- Introduced a module for IntelMQ related datatypes `intelmq.lib.datatypes` which for now only contains an Enum listing the four bot types
+- Added a `bottype` attribute to CollectorBot, ParserBot, ExpertBot, OutputBot
 
 ### Development
 
 ### Data Format
 
 ### Bots
+- Set the parent class of all bots to the correct bot class
+
 #### Collectors
 
 #### Parsers

intelmq/bots/experts/abusix/expert.py

@@ -8,7 +8,7 @@
 RIPE abuse contacts resolving through DNS TXT queries
 '''
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 from ._lib import Abusix
 
@@ -18,7 +18,7 @@
     querycontacts = None
 
 
-class AbusixExpertBot(Bot):
+class AbusixExpertBot(ExpertBot):
     """Add abuse contact information from the Abusix online service for source and destination IP address"""
 
     def init(self):

intelmq/bots/experts/aggregate/expert.py

@@ -8,12 +8,12 @@
 from datetime import datetime, timedelta
 import time
 import json
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import parse_relative
 from intelmq.lib.mixins import CacheMixin
 
 
-class AggregateExpertBot(Bot, CacheMixin):
+class AggregateExpertBot(ExpertBot, CacheMixin):
     """Aggregation expert bot"""
 
     fields: str = "classification.type, classification.identifier"

intelmq/bots/experts/asn_lookup/expert.py

@@ -11,7 +11,7 @@
 import pathlib
 import requests
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -23,7 +23,7 @@
     pyasn = None
 
 
-class ASNLookupExpertBot(Bot):
+class ASNLookupExpertBot(ExpertBot):
     """Add ASN and netmask information from a local BGP dump"""
     database = None  # TODO: should be pathlib.Path
 

intelmq/bots/experts/csv_converter/expert.py

@@ -5,10 +5,10 @@
 # -*- coding: utf-8 -*-
 import csv
 import io
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class CSVConverterExpertBot(Bot):
+class CSVConverterExpertBot(ExpertBot):
     """Convert data to CSV"""
     fieldnames: str = "time.source,classification.type,source.ip"  # TODO: could maybe be List[str]
     delimiter: str = ','

intelmq/bots/experts/cymru_whois/expert.py

@@ -5,7 +5,7 @@
 # -*- coding: utf-8 -*-
 import json
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.harmonization import IPAddress
 from intelmq.lib.mixins import CacheMixin
 
@@ -14,7 +14,7 @@
 CACHE_KEY = "%d_%s"
 
 
-class CymruExpertBot(Bot, CacheMixin):
+class CymruExpertBot(ExpertBot, CacheMixin):
     """Add ASN, netmask, AS name, country, registry and allocation time from the Cymru Whois DNS service"""
     overwrite = False
     redis_cache_db: int = 5

intelmq/bots/experts/deduplicator/expert.py

@@ -26,11 +26,11 @@
                  system because system will always ignore this key.
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.mixins import CacheMixin
 
 
-class DeduplicatorExpertBot(Bot, CacheMixin):
+class DeduplicatorExpertBot(ExpertBot, CacheMixin):
     """Detection and drop exact duplicate messages. Message hashes are cached in the Redis database"""
     filter_keys: str = "raw,time.observation"  # TODO: could be List[str]
     filter_type: str = "blacklist"

intelmq/bots/experts/do_portal/expert.py

@@ -14,10 +14,10 @@
     requests = None
 
 import intelmq.lib.utils as utils
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class DoPortalExpertBot(Bot):
+class DoPortalExpertBot(ExpertBot):
     """Retrieve abuse contact information for the source IP address from a do-portal instance"""
     mode: str = "append"
     portal_api_key: str = None

intelmq/bots/experts/domain_suffix/expert.py

@@ -14,7 +14,7 @@
 
 import requests.exceptions
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import InvalidArgument
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -28,7 +28,7 @@
 ALLOWED_FIELDS = ['fqdn', 'reverse_dns']
 
 
-class DomainSuffixExpertBot(Bot):
+class DomainSuffixExpertBot(ExpertBot):
     """Extract the domain suffix from a domain and save it in the the domain_suffix field. Requires a local file with valid domain suffixes"""
     field: str = None
     suffix_file: str = None  # TODO: should be pathlib.Path

intelmq/bots/experts/domain_valid/expert.py

@@ -17,13 +17,13 @@
 
 import requests.exceptions
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError, ConfigurationError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
 
 
-class DomainValidExpertBot(Bot):
+class DomainValidExpertBot(ExpertBot):
     domain_field: str = 'source.fqdn'
     tlds_domains_list: str = '/opt/intelmq/var/lib/bots/domain_valid/tlds-alpha-by-domain.txt'
 

intelmq/bots/experts/field_reducer/expert.py

@@ -7,11 +7,11 @@
 Reducer bot
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.message import Event
 
 
-class FieldReducerExpertBot(Bot):
+class FieldReducerExpertBot(ExpertBot):
     """Remove fields from events"""
     type = None
     keys = None

intelmq/bots/experts/filter/expert.py

@@ -10,11 +10,11 @@
 import pytz
 from dateutil import parser
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import parse_relative, TIMESPANS
 
 
-class FilterExpertBot(Bot):
+class FilterExpertBot(ExpertBot):
     """Filter events, supports named paths for splitting the message flow"""
 
     _message_processed_verb = 'Forwarded'

intelmq/bots/experts/format_field/expert.py

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 # -*- coding: utf-8 -*-
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
-class FormatFieldExpertBot(Bot):
+class FormatFieldExpertBot(ExpertBot):
     """Perform string method operations on column values"""
     new_value       = ""
     old_value       = ""

intelmq/bots/experts/generic_db_lookup/expert.py

@@ -7,11 +7,11 @@
 Generic DB Lookup
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.mixins import SQLMixin
 
 
-class GenericDBLookupExpertBot(Bot, SQLMixin):
+class GenericDBLookupExpertBot(ExpertBot, SQLMixin):
     """Fetche data from a database"""
     database: str = "intelmq"
     engine: str = "<postgresql OR sqlite>"

intelmq/bots/experts/geohash/expert.py

@@ -8,15 +8,15 @@
 https://pypi.org/project/geolib/
 https://github.com/joyanujoy/geolib
 '''
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 try:
     from geolib import geohash
 except ImportError:
     geohash = None
 
 
-class GeohashExpertBot(Bot):
+class GeohashExpertBot(ExpertBot):
     """Compute the geohash from longitude/latitude information, save it to extra.(source|destination)"""
     overwrite: bool = False
     precision: int = 7

intelmq/bots/experts/gethostbyname/expert.py

@@ -24,12 +24,12 @@
 """
 import socket
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.harmonization import URL
 from intelmq.lib.exceptions import InvalidArgument
 
 
-class GethostbynameExpertBot(Bot):
+class GethostbynameExpertBot(ExpertBot):
     """Resolve the IP address for the FQDN"""
     fallback_to_url: bool = True
     gaierrors_to_ignore = ()

intelmq/bots/experts/http/expert_content.py

@@ -6,11 +6,11 @@
 """
 from typing import List
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import create_request_session
 
 
-class HttpContentExpertBot(Bot):
+class HttpContentExpertBot(ExpertBot):
     """
     Test if a given string is part of the content for a given URL
 

intelmq/bots/experts/http/expert_status.py

@@ -6,11 +6,11 @@
 """
 from typing import List
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import create_request_session
 
 
-class HttpStatusExpertBot(Bot):
+class HttpStatusExpertBot(ExpertBot):
     """
     Fetch the HTTP Status for a given URL
 

intelmq/bots/experts/idea/expert.py

@@ -11,7 +11,7 @@
 from urllib.parse import quote_plus
 from uuid import uuid4
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 
 
 def quot(s):
@@ -26,7 +26,7 @@ def addr6(s):
     return s if ":" in s else None
 
 
-class IdeaExpertBot(Bot):
+class IdeaExpertBot(ExpertBot):
     """Convert events into the IDEA format"""
     test_mode: bool = False
 

intelmq/bots/experts/lookyloo/expert.py

@@ -4,7 +4,7 @@
 
 # -*- coding: utf-8 -*-
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 
 try:
@@ -13,7 +13,7 @@
     pylookyloo = None
 
 
-class LookyLooExpertBot(Bot):
+class LookyLooExpertBot(ExpertBot):
     """ LookyLoo expert bot for automated website screenshots """
     instance_url: str = "http://localhost:5100/"
 

intelmq/bots/experts/maxmind_geoip/expert.py

@@ -14,7 +14,7 @@
 import requests
 import tarfile
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 from intelmq.lib.utils import get_bots_settings, create_request_session
 from intelmq.bin.intelmqctl import IntelMQController
@@ -25,7 +25,7 @@
     geoip2 = None
 
 
-class GeoIPExpertBot(Bot):
+class GeoIPExpertBot(ExpertBot):
     """Add geolocation information from a local MaxMind database to events (country, city, longitude, latitude)"""
     database: str = "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb"  # TODO: should be pathlib.Path
     license_key: str = "<insert Maxmind license key>"

intelmq/bots/experts/mcafee/expert_mar.py

@@ -24,11 +24,11 @@
     MarClient = None
 
 # imports for additional libraries and intelmq
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 
 
-class MARExpertBot(Bot):
+class MARExpertBot(ExpertBot):
     """Query connections to IP addresses to the given destination within the local environment using McAfee Active Response queries"""
     dxl_config_file: str = "<insert /path/to/dxlclient.config>"  # TODO: should be pathlib.Path
     lookup_type: str = "<Hash|DestSocket|DestIP|DestFQDN>"

intelmq/bots/experts/misp/expert.py

@@ -13,7 +13,7 @@
 """
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.exceptions import MissingDependencyError
 
 try:
@@ -22,7 +22,7 @@
     ExpandedPyMISP = None
 
 
-class MISPExpertBot(Bot):
+class MISPExpertBot(ExpertBot):
     """Looking up the IP address in MISP instance and retrieve attribute and event UUIDs"""
     misp_key: str = "<insert MISP Authkey>"
     misp_url: str = "<insert url of MISP server (with trailing '/')>"

intelmq/bots/experts/modify/expert.py

@@ -9,7 +9,7 @@
 import re
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import load_configuration
 
 
@@ -37,7 +37,7 @@ def __getitem__(self, key):
         return self.match.group(key)
 
 
-class ModifyExpertBot(Bot):
+class ModifyExpertBot(ExpertBot):
     """Perform arbitrary changes to event's fields based on regular-expression-based rules on different values. See the bot's documentation for some examples"""
     case_sensitive: bool = True
     configuration_path: str = "/opt/intelmq/var/lib/bots/modify/modify.conf"  # TODO: should be pathlib.Path

intelmq/bots/experts/national_cert_contact_certat/expert.py

@@ -19,7 +19,7 @@
 &sep={TAB, comma, semicolon, pipe}  	Separator for the (output) CSV format
 """
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import ExpertBot
 from intelmq.lib.utils import create_request_session
 from intelmq.lib.exceptions import MissingDependencyError
 
@@ -32,7 +32,7 @@
 URL = 'https://contacts.cert.at/cgi-bin/abuse-nationalcert.pl'
 
 
-class NationalCERTContactCertATExpertBot(Bot):
+class NationalCERTContactCertATExpertBot(ExpertBot):
     """Add country and abuse contact information from the CERT.at national CERT Contact Database. Set filter to true if you want to filter out events for Austria. Set overwrite_cc to true if you want to overwrite an existing country code value"""
     filter: bool = False
     http_verify_cert: bool = True