From a7fdc194f9dd19cda49d185fdc7450d30bb4a48d Mon Sep 17 00:00:00 2001 From: Benjamin Gerber Date: Tue, 11 Jun 2024 15:35:27 +0200 Subject: [PATCH] Fix parsing of the ini file to setup loggers Done for sqlalchemylogger, but actually the logger would try to write logs (process) on another process than the logs collecteur (emit) which can't work. The setup of sqlalchemy logger must be done manually in the project and not via the ini file. --- BREAKING_CHANGES.md | 7 +++++ acceptance_tests/app/production.ini | 4 +-- c2cwsgiutils/__init__.py | 19 ++++++----- c2cwsgiutils/sqlalchemylogger/README.md | 42 +++++++++++++++++-------- production.ini | 4 +-- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/BREAKING_CHANGES.md b/BREAKING_CHANGES.md index 6833eedaa..9ad0201ae 100644 --- a/BREAKING_CHANGES.md +++ b/BREAKING_CHANGES.md @@ -1,5 +1,12 @@ # Changelog +## Release 6.1 + + - The `handlers` in the `.ini` files don't support `args` anymore. You must use `kwargs` + arguments. Example `args = (sys.stdout,)` becomes `kwargs = {'stream': 'ext://sys.stdout'}`. + - SqlAlchemy logger must now be instantiated by your app's `main` method and not by your + `.ini` file. Read the example in the sqlalchemylogger folder. + ## Release 6.0 - The stats will not anymore be published on StatsD, use Prometheus client instead. diff --git a/acceptance_tests/app/production.ini b/acceptance_tests/app/production.ini index 578d52bbd..ea8d86138 100644 --- a/acceptance_tests/app/production.ini +++ b/acceptance_tests/app/production.ini @@ -77,13 +77,13 @@ qualname = c2cwsgiutils_app [handler_console] class = logging.StreamHandler -args = (sys.stdout,) +kwargs = {'stream': 'ext://sys.stdout'} level = NOTSET formatter = generic [handler_json] class = c2cwsgiutils.pyramid_logging.JsonLogHandler -args = (sys.stdout,) +kwargs = {'stream': 'ext://sys.stdout'} level = NOTSET formatter = generic diff --git a/c2cwsgiutils/__init__.py b/c2cwsgiutils/__init__.py index 467ed2644..32d13aa0f 100644 --- a/c2cwsgiutils/__init__.py +++ b/c2cwsgiutils/__init__.py @@ -2,6 +2,7 @@ import logging import os import re +import ast import sys from configparser import SectionProxy from typing import Any @@ -35,20 +36,24 @@ def _create_handlers(config: configparser.ConfigParser) -> dict[str, Any]: stream_re = re.compile(r"\((.*?),\)") for hh in handlers: block = config[f"handler_{hh}"] - stream_match = stream_re.match(block["args"]) - if stream_match is None: - raise Exception(f"Could not parse args of handler {hh}") # pylint: disable=broad-exception-raised - args = stream_match.groups()[0] + if "args" in block: + raise ValueError(f"Can not parse args of handlers {hh}, use kwargs instead.") c = block["class"] if "." not in c: # classes like StreamHandler does not need the prefix in the ini so we add it here c = f"logging.{c}" conf = { - "class": c, - "stream": f"ext://{args}", # like ext://sys.stdout + "class": c, } + if "level" in block: + conf["level"] = block["level"] if "formatter" in block: conf["formatter"] = block["formatter"] + if "filters" in block: + conf["filters"] = block["filters"] + if "kwargs" in block: + kwargs = ast.literal_eval(block["kwargs"]) + conf.update(kwargs) d_handlers[hh] = conf return d_handlers @@ -57,7 +62,7 @@ def _filter_logger(block: SectionProxy) -> dict[str, Any]: out: dict[str, Any] = {"level": block["level"]} handlers = block.get("handlers", "") if handlers != "": - out["handlers"] = [block["handlers"]] + out["handlers"] = [k.strip() for k in block["handlers"].split(",")] return out diff --git a/c2cwsgiutils/sqlalchemylogger/README.md b/c2cwsgiutils/sqlalchemylogger/README.md index 53bf99c72..86c64cf97 100644 --- a/c2cwsgiutils/sqlalchemylogger/README.md +++ b/c2cwsgiutils/sqlalchemylogger/README.md @@ -2,22 +2,38 @@ This module is used to ship logging records to an SQL database. Currently only `sqlite` and `postgres_psycopg2` are fully supported. -To add the logger in a pyramid ini file use something like: +To add the handler, setup it directly in your app's main function. You +can add it to an existing logger (setup in you `.ini` file), +or create a new logger by calling the `logging.getlogger` method. +```python +import logging +from c2cwsgiutils.sqlalchemylogger.handlers import SQLAlchemyHandler + +def _setup_sqlalchemy_logger(): + """ + Setup sqlalchemy logger. + """ + logger = logging.getLogger("A_LOGGER") + handler = SQLAlchemyHandler( + sqlalchemy_url={ + # "url": "sqlite:///logger_db.sqlite3", + "url": "postgresql://postgres:password@localhost:5432/test", + "tablename": "test", + "tableargs": {"schema": "xyz"}, + }, + does_not_contain_expression="curl", + ) + logger.addHandler(handler) + +def main(_, **settings): + _setup_sqlalchemy_logger () +... ``` -[handlers] -keys = sqlalchemy_logger - -[handler_sqlalchemy_logger] -class = c2cwsgiutils.sqlalchemylogger.handlers.SQLAlchemyHandler -#args = ({'url':'sqlite:///logger_db.sqlite3','tablename':'test'},'curl') -args = ({'url':'postgresql://postgres:password@localhost:5432/test','tablename':'test','tableargs': {'schema':'xyz'}},'curl') -level = NOTSET -formatter = generic -propagate = 0 -``` +Do not set up this sqlalchemy logger in you `.ini` file directly. +It won't work (multi process issue). -if the credentials given in `args = ` section are sufficient, the handler will +if the given credentials are sufficient, the handler will create the DB, schema and table it needs directly. In the above example the second parameter provided `'curl'` is a negative diff --git a/production.ini b/production.ini index ba222649a..1d177ea70 100644 --- a/production.ini +++ b/production.ini @@ -31,13 +31,13 @@ qualname = c2cwsgiutils [handler_console] class = logging.StreamHandler -args = (sys.stdout,) +kwargs = {'stream': 'ext://sys.stdout'} level = NOTSET formatter = generic [handler_json] class = c2cwsgiutils.pyramid_logging.JsonLogHandler -args = (sys.stdout,) +kwargs = {'stream': 'ext://sys.stdout'} level = NOTSET formatter = generic