diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f5fa23f9..3cd715f41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ CHANGELOG ### Core - `intelmq.lib.bot.CollectorBot`: Fixed an issue with within the `new_report` function, which re-loads the harmonization file after a new incoming dataset, which leads to CPU drain and decreased performance (PR#2106 by Sebastian Waldbauer, fixes #2098). +- `intelmq.lib.bot.Bot`: Make private members `__is_multithreadable` and `__collector_empty_process` protected members `_is_multithreadable` and `_collector_empty_process` to make them easily modifiable by Bot classes (PR#2109 by Sebastian Wagner, fixes #2108). + Also affected and adapted bots by this change are: + - `intelmq.bots.collectors.api.collector_api` + - `intelmq.bots.collectors.stomp.collector` + - `intelmq.bots.experts.splunk_saved_search.expert` + - `intelmq.bots.experts.threshold.expert` + - `intelmq.bots.outputs.file.output` + - `intelmq.bots.outputs.misp.output_api` + - `intelmq.bots.outputs.misp.output_feed` + - `intelmq.bots.outputs.tcp.output` + - `intelmq.bots.outputs.udp.output` ### Bots #### Experts diff --git a/intelmq/bots/collectors/api/collector_api.py b/intelmq/bots/collectors/api/collector_api.py index 6f77634a2..c950c7cd6 100644 --- a/intelmq/bots/collectors/api/collector_api.py +++ b/intelmq/bots/collectors/api/collector_api.py @@ -37,9 +37,9 @@ class APICollectorBot(CollectorBot): """Collect data by exposing a HTTP API interface""" name: str = "API" port: int = 5000 - __collector_empty_process: bool = True + _collector_empty_process: bool = True provider: str = "APICollector" - __is_multithreadable: bool = False + _is_multithreadable: bool = False use_socket = False socket_path = '/tmp/imq_api_default_socket' _server: Optional['HTTPServer'] = None diff --git a/intelmq/bots/collectors/stomp/collector.py b/intelmq/bots/collectors/stomp/collector.py index a74fccdfa..0c9f60456 100644 --- a/intelmq/bots/collectors/stomp/collector.py +++ b/intelmq/bots/collectors/stomp/collector.py @@ -69,7 +69,7 @@ class StompCollectorBot(CollectorBot): ssl_client_certificate_key: str = 'client.key' # TODO pathlib.Path heartbeat: int = 6000 - __collector_empty_process: bool = True + _collector_empty_process: bool = True __conn = False # define here so shutdown method can check for it def init(self): diff --git a/intelmq/bots/experts/splunk_saved_search/expert.py b/intelmq/bots/experts/splunk_saved_search/expert.py index 34f9d4ad2..2d8aba9e1 100644 --- a/intelmq/bots/experts/splunk_saved_search/expert.py +++ b/intelmq/bots/experts/splunk_saved_search/expert.py @@ -82,7 +82,7 @@ class SplunkSavedSearchBot(Bot): search_parameters = {"event field": "search parameter"} url: str = None - __is_multithreadable = False + _is_multithreadable = False def init(self): if requests is None: diff --git a/intelmq/bots/experts/threshold/expert.py b/intelmq/bots/experts/threshold/expert.py index 3d96095dc..c1bdb78ae 100644 --- a/intelmq/bots/experts/threshold/expert.py +++ b/intelmq/bots/experts/threshold/expert.py @@ -68,7 +68,7 @@ class ThresholdExpertBot(Bot, CacheMixin): _message_processed_verb = 'Forwarded' - __is_multithreadable = False + _is_multithreadable = False bypass = False def init(self): diff --git a/intelmq/bots/outputs/file/output.py b/intelmq/bots/outputs/file/output.py index 449125b10..d8dd37841 100644 --- a/intelmq/bots/outputs/file/output.py +++ b/intelmq/bots/outputs/file/output.py @@ -22,7 +22,7 @@ class FileOutputBot(OutputBot): message_jsondict_as_string: bool = False message_with_type: bool = False single_key: bool = False - __is_multithreadable = False + _is_multithreadable = False def init(self): # needs to be done here, because in process() FileNotFoundError handling we call init(), diff --git a/intelmq/bots/outputs/misp/output_api.py b/intelmq/bots/outputs/misp/output_api.py index 0f22aae4b..28a971364 100644 --- a/intelmq/bots/outputs/misp/output_api.py +++ b/intelmq/bots/outputs/misp/output_api.py @@ -95,7 +95,7 @@ class MISPAPIOutputBot(OutputBot): misp_url: str = None significant_fields: list = [] - _Bot__is_multithreadable = False + _is_multithreadable = False def init(self): if pymisp is None and import_fail_reason == 'syntax': diff --git a/intelmq/bots/outputs/misp/output_feed.py b/intelmq/bots/outputs/misp/output_feed.py index 5b4839474..e5e836552 100644 --- a/intelmq/bots/outputs/misp/output_feed.py +++ b/intelmq/bots/outputs/misp/output_feed.py @@ -35,7 +35,7 @@ class MISPFeedOutputBot(OutputBot): misp_org_name = None misp_org_uuid = None output_dir: str = "/opt/intelmq/var/lib/bots/mispfeed-output" # TODO: should be path - __is_multithreadable: bool = False + _is_multithreadable: bool = False @staticmethod def check_output_dir(dirname): diff --git a/intelmq/bots/outputs/tcp/output.py b/intelmq/bots/outputs/tcp/output.py index e7b92ec92..57389ede4 100644 --- a/intelmq/bots/outputs/tcp/output.py +++ b/intelmq/bots/outputs/tcp/output.py @@ -25,7 +25,7 @@ class TCPOutputBot(Bot): port: int = None separator: str = None - __is_multithreadable = False + _is_multithreadable = False def init(self): self.to_intelmq = self.counterpart_is_intelmq diff --git a/intelmq/bots/outputs/udp/output.py b/intelmq/bots/outputs/udp/output.py index b59e4f4f3..a48a05c31 100644 --- a/intelmq/bots/outputs/udp/output.py +++ b/intelmq/bots/outputs/udp/output.py @@ -19,7 +19,7 @@ class UDPOutputBot(Bot): udp_host: str = "localhost" udp_port: int = None - __is_multithreadable = False + _is_multithreadable = False def init(self): self.delimiter = self.field_delimiter diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index c0069bb73..78c9fcf5e 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -104,9 +104,9 @@ class Bot(object): # True for (non-main) threads of a bot instance is_multithreaded: bool = False # True if the bot is thread-safe and it makes sense - __is_multithreadable: bool = True + _is_multithreadable: bool = True # Collectors with an empty process() should set this to true, prevents endless loops (#1364) - __collector_empty_process: bool = False + _collector_empty_process: bool = False def __init__(self, bot_id: str, start: bool = False, sighup_event=None, disable_multithreading: bool = None): @@ -158,11 +158,11 @@ def __init__(self, bot_id: str, start: bool = False, sighup_event=None, broker = self.source_pipeline_broker.title() if broker != 'Amqp': - self.__is_multithreadable = False + self._is_multithreadable = False """ Multithreading """ if (self.instances_threads > 1 and not self.is_multithreaded and - self.__is_multithreadable and not disable_multithreading): + self._is_multithreadable and not disable_multithreading): self.logger.handlers = [] num_instances = int(self.instances_threads) instances = [] @@ -189,7 +189,7 @@ def handle_sighup_signal_threading(signum: int, thread.join() return elif (getattr(self, 'instances_threads', 1) > 1 and - not self.__is_multithreadable): + not self._is_multithreadable): self.logger.error('Multithreading is configured, but is not ' 'available for this bot. Look at the FAQ ' 'for a list of reasons for this. ' @@ -435,7 +435,7 @@ def start(self, starting: bool = True, error_on_pipeline: bool = True, if do_rate_limit: if self.rate_limit and self.run_mode != 'scheduled': self.__sleep() - if self.__collector_empty_process and self.run_mode != 'scheduled': + if self._collector_empty_process and self.run_mode != 'scheduled': self.__sleep(1, log=False) self.__stats() @@ -1152,7 +1152,7 @@ class CollectorBot(Bot): Does some sanity checks on message sending. """ - __is_multithreadable: bool = False + _is_multithreadable: bool = False name: Optional[str] = None accuracy: int = 100 code: Optional[str] = None