From d71a65added056d55a2c6c53da84477da1edad47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 15:29:39 +0200 Subject: [PATCH 1/9] Introduce karton debug --- .gitignore | 2 ++ karton/core/base.py | 12 ++++++++++-- karton/core/karton.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 90dbfefb..95637222 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ karton.ini config.ini docs/_build .mypy_cache +build +karton_core.egg-info/ diff --git a/karton/core/base.py b/karton/core/base.py index 3d8420c5..42fe6674 100644 --- a/karton/core/base.py +++ b/karton/core/base.py @@ -1,4 +1,5 @@ import abc +import os import argparse import logging import textwrap @@ -8,7 +9,7 @@ from .__version__ import __version__ from .backend import KartonBackend, KartonServiceInfo from .config import Config -from .logger import KartonLogHandler +from .logger import KartonLogHandler, KartonLogDebugHandler from .task import Task from .utils import HardShutdownInterrupt, StrictClassMethod, graceful_killer @@ -44,6 +45,11 @@ def __init__( if self.config.has_option("karton", "identity"): self.identity = self.config.get("karton", "identity") + self.debug = self.config.getboolean("karton", "debug") + + if self.debug: + self.identity += "-" + os.urandom(4).hex() + "-dev" + self.service_info = None if self.identity is not None and self.with_service_info: self.service_info = KartonServiceInfo( @@ -101,7 +107,9 @@ def setup_logger(self, level: Optional[Union[str, int]] = None) -> None: logging.Formatter("[%(asctime)s][%(levelname)s] %(message)s") ) logger.addHandler(stream_handler) - logger.addHandler(self._log_handler) + + if not self.debug: + logger.addHandler(self._log_handler) @property def log_handler(self) -> KartonLogHandler: diff --git a/karton/core/karton.py b/karton/core/karton.py index 0f46c36e..ff20ca2a 100644 --- a/karton/core/karton.py +++ b/karton/core/karton.py @@ -123,7 +123,7 @@ def __init__( self.persistent = self.config.getboolean( "karton", "persistent", self.persistent - ) + ) and not self.debug self.task_timeout = self.config.getint("karton", "task_timeout") self.current_task: Optional[Task] = None self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = [] From fe771892a7c68295ce1aa341ae07d11c675c4a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 15:31:06 +0200 Subject: [PATCH 2/9] Remove dead import --- karton/core/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karton/core/base.py b/karton/core/base.py index 42fe6674..9ab03e12 100644 --- a/karton/core/base.py +++ b/karton/core/base.py @@ -9,7 +9,7 @@ from .__version__ import __version__ from .backend import KartonBackend, KartonServiceInfo from .config import Config -from .logger import KartonLogHandler, KartonLogDebugHandler +from .logger import KartonLogHandler from .task import Task from .utils import HardShutdownInterrupt, StrictClassMethod, graceful_killer From b80bd5461e3f5df88013674fcf73a472588d3715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 15:42:45 +0200 Subject: [PATCH 3/9] Add cli configuration --- karton/core/base.py | 8 +++++++- karton/core/karton.py | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/karton/core/base.py b/karton/core/base.py index 9ab03e12..5501becc 100644 --- a/karton/core/base.py +++ b/karton/core/base.py @@ -162,6 +162,9 @@ def args_parser(cls) -> argparse.ArgumentParser: "--identity", help="Alternative identity for Karton service" ) parser.add_argument("--log-level", help="Logging level of Karton logger") + parser.add_argument( + "--debug", help="Enable debugging mode", action="store_true", default=None + ) return parser @classmethod @@ -174,7 +177,10 @@ def config_from_args(cls, config: Config, args: argparse.Namespace) -> None: """ config.load_from_dict( { - "karton": {"identity": args.identity}, + "karton": { + "identity": args.identity, + "debug": args.debug, + }, "logging": {"level": args.log_level}, } ) diff --git a/karton/core/karton.py b/karton/core/karton.py index ff20ca2a..88fc0583 100644 --- a/karton/core/karton.py +++ b/karton/core/karton.py @@ -121,9 +121,10 @@ def __init__( if self.filters is None: raise ValueError("Cannot bind consumer on Empty binds") - self.persistent = self.config.getboolean( - "karton", "persistent", self.persistent - ) and not self.debug + self.persistent = ( + self.config.getboolean("karton", "persistent", self.persistent) + and not self.debug + ) self.task_timeout = self.config.getint("karton", "task_timeout") self.current_task: Optional[Task] = None self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = [] From 5de1849ea14d0f5ba57dc0cc315e9b0b783300fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 15:44:39 +0200 Subject: [PATCH 4/9] Sort imports --- karton/core/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karton/core/base.py b/karton/core/base.py index 5501becc..d4970883 100644 --- a/karton/core/base.py +++ b/karton/core/base.py @@ -1,7 +1,7 @@ import abc -import os import argparse import logging +import os import textwrap from contextlib import contextmanager from typing import Optional, Union, cast From ee5a6e63f813d4f074e1f256f16146d637616a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 16:13:25 +0200 Subject: [PATCH 5/9] Mention debug in docs --- docs/service_configuration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/service_configuration.rst b/docs/service_configuration.rst index 79b01170..50ab9695 100644 --- a/docs/service_configuration.rst +++ b/docs/service_configuration.rst @@ -48,6 +48,7 @@ Common Karton configuration fields are listed below: [redis] socket_timeout Socket timeout for Redis operations in seconds (default: 30, use 0 to turn off if timeout doesn't work properly) [karton] identity Karton service identity override (overrides the name provided in class / constructor arguments) [karton] persistent Karton service queue persistency override + [karton] debug Karton debug mode for service development [karton] task_timeout Karton service task execution timeout in seconds. Useful if your service sometimes hangs. Karton will schedule SIGALRM if this value is set. [logging] level Logging level for Karton service logger (default: INFO) [signaling] status Turns on producing of 'karton.signaling.status' tasks, signalling the task start and finish events by Karton service (default: 0, off) @@ -90,6 +91,7 @@ All settings can be set using command-line. --identity IDENTITY Alternative identity for Karton service --log-level LOG_LEVEL Logging level of Karton logger + --debug Enable debugging mode --setup-bucket Create missing bucket in S3 storage --disable-gc Do not run GC in this instance --disable-router Do not run task routing in this instance From 5ab975cf3cb8a2d749d3cd20464610ce0406fe68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 16:32:09 +0200 Subject: [PATCH 6/9] Add more docs --- docs/advanced_concepts.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/advanced_concepts.rst b/docs/advanced_concepts.rst index 224329cb..f8d2854e 100644 --- a/docs/advanced_concepts.rst +++ b/docs/advanced_concepts.rst @@ -230,3 +230,19 @@ The simplest way to do that is to perform all of these actions synchronously, in # If analysis has been finished: get the results and process them analysis = sandbox.get_results(analysis_id) self.process_results(analysis) + + +Karton debug mode +----------------- + +During your karton services development endeavours you'll often have the urge to test them out on the production environment. + +While this is totally fine you have to be careful not to disrupt the production services by consuming the tasks meant for them. + +Karton debug mode was crafted especially for this purpose. It adds a random suffix to the service identity to create a new, non-conflicting task queue. +It also permanently sets the consumer persistence to False and disables log forwarding. + +You can enable it by setting: +- :code:`KARTON_KARTON_DEBUG` environment value +- :code:`[debug]` parameter in the :code:`[karton]` config section +- :code:`--debug` command-line parameter From 622b4308ce3eadb8f137a4588036f5a8ddbcba7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Thu, 13 Jul 2023 18:19:18 +0200 Subject: [PATCH 7/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jarosław Jedynak --- docs/advanced_concepts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/advanced_concepts.rst b/docs/advanced_concepts.rst index f8d2854e..cdf6c4da 100644 --- a/docs/advanced_concepts.rst +++ b/docs/advanced_concepts.rst @@ -237,12 +237,12 @@ Karton debug mode During your karton services development endeavours you'll often have the urge to test them out on the production environment. -While this is totally fine you have to be careful not to disrupt the production services by consuming the tasks meant for them. +While this is totally fine, you have to be careful not to disrupt the production services by consuming the tasks meant for them. In most cases you also don't want to propagate logs from your experiments to the production log pipeline. Karton debug mode was crafted especially for this purpose. It adds a random suffix to the service identity to create a new, non-conflicting task queue. It also permanently sets the consumer persistence to False and disables log forwarding. You can enable it by setting: -- :code:`KARTON_KARTON_DEBUG` environment value -- :code:`[debug]` parameter in the :code:`[karton]` config section +- :code:`KARTON_KARTON_DEBUG` environment value to "1" +- :code:`[debug]` parameter to `1` in the :code:`[karton]` config section - :code:`--debug` command-line parameter From b52ff0627ce8526506e97746608c5899585d5cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Praszmo?= Date: Mon, 17 Jul 2023 11:44:56 +0200 Subject: [PATCH 8/9] Change default to false --- karton/core/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karton/core/base.py b/karton/core/base.py index d4970883..846509ee 100644 --- a/karton/core/base.py +++ b/karton/core/base.py @@ -45,7 +45,7 @@ def __init__( if self.config.has_option("karton", "identity"): self.identity = self.config.get("karton", "identity") - self.debug = self.config.getboolean("karton", "debug") + self.debug = self.config.getboolean("karton", "debug", False) if self.debug: self.identity += "-" + os.urandom(4).hex() + "-dev" From 1d28c23f626cec62fb4a393a12830164b8542837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Srokosz?= Date: Thu, 20 Jul 2023 17:47:38 +0200 Subject: [PATCH 9/9] Apply suggestions from code review --- docs/advanced_concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced_concepts.rst b/docs/advanced_concepts.rst index cdf6c4da..3d75953d 100644 --- a/docs/advanced_concepts.rst +++ b/docs/advanced_concepts.rst @@ -244,5 +244,5 @@ It also permanently sets the consumer persistence to False and disables log forw You can enable it by setting: - :code:`KARTON_KARTON_DEBUG` environment value to "1" -- :code:`[debug]` parameter to `1` in the :code:`[karton]` config section +- :code:`debug` parameter to `1` in the :code:`[karton]` config section - :code:`--debug` command-line parameter