Add same possibility on Prometheus as with statsd

Be able to easily add
- a gauge
- a counter
- a timer (elapsed)

.prospector.yaml

@@ -15,6 +15,10 @@ pylint:
       - lxml
   disable:
     - too-many-return-statements
+    - too-many-arguments
+    - too-many-branches
+    - too-many-instance-attributes
+    - too-few-public-methods
     - global-statement
     - line-too-long
     - import-outside-toplevel
@@ -24,8 +28,6 @@ pylint:
     - no-self-use
     - import-error
     - superfluous-parens
-    - too-few-public-methods
-    - too-many-arguments
     - ungrouped-imports
     - unused-argument
     - use-symbolic-message-instead

BREAKING_CHANGES.mg → BREAKING_CHANGES.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Release 5.3
+
+- The stats will not anymore be published on StatsD, use Prometheus client instead.
+  stats is no more initialized by default, you need to call `c2cwsgiutils.stats.init_backends(settings)` and
+  `c2cwsgiutils.config.include(metrics.includeme)` to initialize it.
+
 ## Release 5.2
 
 - `c2cwsgiutils_run` is completely removes (not used from 5.0).

Dockerfile

@@ -70,7 +70,9 @@ RUN --mount=type=cache,target=/root/.cache \
   python3 -m pip install --disable-pip-version-check --no-deps --editable=. \
   && python3 -m compileall -q \
   && python3 -m compileall /usr/local/lib/python3.* /usr/lib/python3.* . -q \
-  && python3 -c 'import c2cwsgiutils'
+  && python3 -c 'import c2cwsgiutils' \
+  && mkdir -p /prometheus-metrics \
+  && chmod a+rwx /prometheus-metrics
 
 ENV C2C_BASE_PATH=/c2c \
   C2C_REDIS_URL= \

Makefile

@@ -79,9 +79,9 @@ pull: ## Pull the Docker images
 	for image in `find -name "docker-compose*.yaml" | xargs grep --no-filename "image:" | awk '{print $$2}' | sort -u | grep -v $(DOCKER_BASE) | grep -v rancher`; do docker pull $$image; done
 
 .PHONY: run
-run: build_test_app ## Run the test application
+run: build_test_app build_docker ## Run the test application
 	# cp acceptance_tests/tests/docker-compose.override.sample.yaml acceptance_tests/tests/docker-compose.override.yaml
-	cd acceptance_tests/tests/; TEST_IP=172.17.0.1 docker-compose up
+	cd acceptance_tests/tests/; docker-compose up --detach
 
 .PHONY: mypy_local
 mypy_local: .venv/timestamp

README.md

@@ -43,28 +43,28 @@ You should install `c2cwsgiutils` with the tool you use to manage your pip depen
 
 In the `Dockerfile` you should add the following lines:
 
-```
+```dockerfile
 # Generate the version file.
 RUN c2cwsgiutils-genversion $(git rev-parse HEAD)
 
 CMD ["gunicorn", "--paste=/app/production.ini"]
 
 # Default values for the environment variables
 ENV \
-    DEVELOPMENT=0 \
-    SQLALCHEMY_POOL_RECYCLE=30 \
-    SQLALCHEMY_POOL_SIZE=5 \
-    SQLALCHEMY_MAX_OVERFLOW=25 \
-    SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
-    SQLALCHEMY_SLAVE_POOL_SIZE=5 \
-    SQLALCHEMY_SLAVE_MAX_OVERFLOW=25\
-    LOG_TYPE=console \
-    OTHER_LOG_LEVEL=WARNING \
-    GUNICORN_LOG_LEVEL=WARNING \
-    GUNICORN_ACCESS_LOG_LEVEL=INFO \
-    SQL_LOG_LEVEL=WARNING \
-    C2CWSGIUTILS_LOG_LEVEL=WARNING \
-    LOG_LEVEL=INFO
+  DEVELOPMENT=0 \
+  SQLALCHEMY_POOL_RECYCLE=30 \
+  SQLALCHEMY_POOL_SIZE=5 \
+  SQLALCHEMY_MAX_OVERFLOW=25 \
+  SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
+  SQLALCHEMY_SLAVE_POOL_SIZE=5 \
+  SQLALCHEMY_SLAVE_MAX_OVERFLOW=25\
+  LOG_TYPE=console \
+  OTHER_LOG_LEVEL=WARNING \
+  GUNICORN_LOG_LEVEL=WARNING \
+  GUNICORN_ACCESS_LOG_LEVEL=INFO \
+  SQL_LOG_LEVEL=WARNING \
+  C2CWSGIUTILS_LOG_LEVEL=WARNING \
+  LOG_LEVEL=INFO
 ```
 
 Add in your `main` function.
@@ -272,6 +272,8 @@ configure a default timeout with the `C2C_REQUESTS_DEFAULT_TIMEOUT` environment
 
 ## Metrics
 
+Deprecated, the Prometheus client is preferred.
+
 To enable and configure the metrics framework, you can use:
 
 - STATS_VIEW (c2c.stats_view): if defined, will enable the stats view `{C2C_BASE_PATH}/stats.json`
@@ -458,14 +460,16 @@ If the `/app/versions.json` exists, a view is added (`{C2C_BASE_PATH}/versions.j
 version of a app. This file is generated by calling the `c2cwsgiutils-genversion [$GIT_TAG] $GIT_HASH`
 command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile) of the WSGI application.
 
-## Metrics
+## Prometheus metrics
+
+Deprecated, the Prometheus client is preferred.
 
 The path `/metrics` provide some metrics for Prometheus.
 By default we have the `smap` `pss`, but we can easily add the `rss`, `size` or your custom settings:
 
 Example:
 
-```
+```python
 from import c2cwsgiutils.metrics import add_provider, Provider, MemoryMapProvider
 
 class CustomProvider(Provider):
@@ -479,6 +483,92 @@ add_provider(MemoryMapProvider('rss'))
 add_provider(CustomProvider())
 ```
 
+## Prometheus client
+
+[Prometheus client](https://github.com/prometheus/client_python) in integrated in c2cwsgiutils.
+
+It will work in multi process mode with the limitation listed in the
+[`prometheus_client` documentation](https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn).
+
+To enable it you should provide the `PROMETHEUS_PORT` environment variable.
+
+We can customize it with the following environment variables:
+
+- `PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
+- `C2C_PROMETHEUS_PACKAGES` the packages that will be present in the version information, default is `c2cwsgiutils,pyramid,gunicorn,sqlalchemy`.
+- `C2C_PROMETHEUS_APPLICATION_PACKAGES` the packages that will be present in the version information as application.
+
+And you should add in your `gunicorn.conf.py`:
+
+```python
+from prometheus_client import multiprocess
+
+
+def on_starting(server):
+    from c2cwsgiutils import prometheus_client
+
+    del server
+
+    prometheus_client.start()
+
+
+def post_fork(server, worker):
+    from c2cwsgiutils import prometheus_client
+
+    del server, worker
+
+    prometheus_client.cleanup()
+
+
+def child_exit(server, worker):
+    del server
+
+    multiprocess.mark_process_dead(worker.pid)
+```
+
+In your `Dockerfile` you should add:
+
+```dockerfile
+RUN mkdir -p /prometheus-metrics \
+    chmod a+rwx /prometheus-metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
+```
+
+### Add custom metric collector
+
+Related to the process.
+
+```python
+from c2cwsgiutils import broadcast, prometheus_client
+
+prometheus_client.MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS.append("prometheus_collector_custom")
+broadcast.subscribe("c2cwsgiutils_prometheus_collect_gc", _broadcast_collector_custom)
+my_custom_collector_instance = MyCustomCollector()
+
+
+def _broadcast_collector_custom() -> List[prometheus_client.SerializedGauge]:
+    """Get the collected GC gauges."""
+
+    return prometheus_client.serialize_collected_data(my_custom_collector_instance)
+```
+
+Related to the host, use that in the `gunicorn.conf.py`:
+
+```python
+def on_starting(server):
+    from c2cwsgiutils import prometheus_client
+
+    del server
+
+    registry = CollectorRegistry()
+    registry.register(MyCollector())
+    prometheus_client.start(registry)
+```
+
+### Usage of metrics
+
+With c2cwsgiutils each instance (Pod) has its own metrics, so we need to aggregate them to have the metrics for the service you probably need to use `sum by (<fields>) (<metric>)` to get the metric (especially for counters).
+
 ## Custom scripts
 
 To have the application initialized in a script you should use the
@@ -531,7 +621,7 @@ have dumps of a few things:
 - memory usage: `{C2C_BASE_PATH}/debug/memory?secret={C2C_SECRET}&limit=30&analyze_type=builtins.dict&python_internals_map=false`
 - object ref: `{C2C_BASE_PATH}/debug/show_refs.dot?secret={C2C_SECRET}&analyze_type=gunicorn.app.wsgiapp.WSGIApplication&analyze_id=12345&max_depth=3&too_many=10&filter=1024&no_extra_info&backrefs`
   `analyze_type` and `analyze_id` should not ve used toogether, you can use it like:
-  ```
+  ```bash
   curl "<URL>" > /tmp/show_refs.dot
   dot -Lg -Tpng /tmp/show_refs.dot > /tmp/show_refs.png
   ```
@@ -592,7 +682,7 @@ client. In production mode, you can still get them by sending the secret defined
 
 If you want to use pyramid_debugtoolbar, you need to disable exception handling and configure it like that:
 
-```
+```ini
 pyramid.includes =
     pyramid_debugtoolbar
 debugtoolbar.enabled = true

acceptance_tests/app/c2cwsgiutils_app/__init__.py

@@ -25,6 +25,7 @@ def main(_, **settings):
     """
     This function returns a Pyramid WSGI application.
     """
+
     config = Configurator(settings=settings, route_prefix="/api")
 
     # Initialize the broadcast view before c2cwsgiutils is initialized. This allows to test the

acceptance_tests/app/gunicorn.conf.py

@@ -4,6 +4,8 @@
 ###
 import os
 
+from prometheus_client import multiprocess
+
 from c2cwsgiutils import get_config_defaults, get_logconfig_dict, get_paste_config
 
 bind = ":8080"
@@ -31,3 +33,25 @@
     print(logconfig_dict)
 
 raw_paste_global_conf = ["=".join(e) for e in get_config_defaults().items()]
+
+
+def on_starting(server):
+    from c2cwsgiutils import prometheus_client
+
+    del server
+
+    prometheus_client.start()
+
+
+def post_fork(server, worker):
+    from c2cwsgiutils import prometheus_client
+
+    del server, worker
+
+    prometheus_client.cleanup()
+
+
+def child_exit(server, worker):
+    del server
+
+    multiprocess.mark_process_dead(worker.pid)

acceptance_tests/tests/.env

@@ -1,3 +1,4 @@
 COMPOSE_PROJECT_NAME=c2cwsgiutils
 SQLALCHEMY_URL=postgresql://www-data:www-data@db:5432/test
 SQLALCHEMY_SLAVE_URL=postgresql://www-data:www-data@db_slave:5432/test
+TEST_IP=172.17.0.1