Add logging with logz

app/__init__.py

@@ -2,25 +2,37 @@
 
 __version__ = "0.1.0.dev0"
 
-from pathlib import Path
-
-import os
-
 import logging
-from logging.handlers import RotatingFileHandler
+import os
+from pathlib import Path
 
 from dotenv import load_dotenv
+from logzio.handler import LogzioHandler
+from logging.handlers import RotatingFileHandler
 
 load_dotenv()
 
 tmp_path = Path(os.getenv("TMPDIR", "/tmp"))
 log_path = tmp_path / "multiqc_api.log"
-logging.basicConfig(
-    level=logging.DEBUG if os.getenv("ENVIRONMENT") == "DEV" else logging.INFO,
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(),
-        RotatingFileHandler(log_path, maxBytes=10_000, backupCount=10),
-    ],
-)
-logging.debug(f"Logging to {log_path}")
+
+logger = logging.getLogger("multiqc_api")
+logger.setLevel(logging.DEBUG)
+
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+fh = RotatingFileHandler(log_path, maxBytes=10_000, backupCount=10)
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(formatter)
+logger.addHandler(fh)
+
+# create console handler with a higher log level
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+logz_handler = LogzioHandler(os.environ['LOGZIO_TOKEN'])
+logz_handler.setFormatter(formatter)
+logz_handler.setLevel(logging.DEBUG)
+logger.addHandler(logz_handler)
+
+logger.info(f"Logging to {log_path}")

app/downloads/daily.py

@@ -31,7 +31,7 @@
 # Load environment variables from the .env file
 load_dotenv()
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("multiqc_api")
 
 SOURCES_DIR = Path(__file__).parent / "sources"
 # Whether we can write back daily.csv and other pulled stats to keep under version control.

app/downloads/sources/daily.csv

@@ -3095,3 +3095,7 @@ date,pip_new,pip_total,bioconda_total,bioconda_new,biocontainers_quay_new,biocon
 2024-02-21,515,77079,72529,241,1958,178849,1,1,0,936,253,,,,,
 2024-02-22,480,1088045,72802,273,3251,348569,2,1,0,938,253,,,3934,47875,1925
 2024-02-23,,,73010,208,,,5,2,0,943,253,,,3956,47895,2076
+2024-03-23,141,80853,29189,128,550,191774,1,1,0,1001,254,,,,,
+2024-03-24,132,80985,29263,74,325,192099,0,0,0,1001,254,,,,,
+2024-03-25,407,81392,29355,92,3099,195198,2,1,0,1003,254,,,,,
+2024-03-26,,,29582,227,,,,,,,,,,4262,48262,1021

app/main.py

@@ -1,4 +1,5 @@
 import logging
+from logzio.handler import LogzioHandler
 
 from typing import List, Dict, Optional
 
@@ -25,7 +26,10 @@
 from app import __version__, db, models
 from app.downloads import daily
 
-logger = logging.getLogger(__name__)
+
+logger = logging.getLogger("multiqc_api")
+
+logger.info("Starting MultiQC API service")
 
 
 app = FastAPI(
@@ -63,6 +67,7 @@ async def startup():
     logger = logging.getLogger("uvicorn.access")
     for h in logger.handlers:
         h.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+
     # Initialise the DB and tables on server startup
     db.create_db_and_tables()
     # Sync latest version tag using GitHub API
@@ -145,7 +150,7 @@ def _log_visit(
                 "is_ci": is_ci,
             }
         )
-        logger.debug(f"Logging visit, total visits: {len(visit_buffer)}")
+        logger.debug(f"Logged visit, total visits in buffer: {len(visit_buffer)}")
 
 
 # Path to a buffer CSV file to persist recent visits before dumping to the database
@@ -199,7 +204,7 @@ def _persist_visits(verbose=False) -> Optional[Response]:
     logger=logger,
 )
 async def persist_visits():
-    return _persist_visits()
+    return _persist_visits(verbose=True)
 
 
 def _summarize_visits(interval="5min") -> Response:

docker-compose.yml

@@ -14,6 +14,7 @@ services:
       GITHUB_TOKEN: $GITHUB_TOKEN
       # Matches the "db" service below
       DATABASE_URL: mysql+pymysql://root:1@db:3306/multiqc
+      LOGZIO_TOKEN: $LOGZIO_TOKEN
     depends_on:
       wait-for-db:
         condition: service_completed_successfully

requirements.txt

@@ -11,3 +11,4 @@ psycopg2-binary
 uvicorn
 python-dotenv
 pypistats
+logzio-python-handler