WIP prometheus

galaxy/api/urls.py

@@ -251,4 +251,5 @@
 urlpatterns = [
     url(r'^$', views.ApiRootView.as_view(), name='api_root_view'),
     url(r'^v1/', include(v1_urls)),
+    url(r'', include('django_prometheus.urls')),
 ]

galaxy/common/metrics.py

@@ -22,9 +22,8 @@
 
 from rest_framework.request import Request
 
-from influxdb import InfluxDBClient
-from influxdb.exceptions import InfluxDBClientError
-from influxdb.exceptions import InfluxDBServerError
+import prometheus_client
+
 
 __all__ = [
     'send',
@@ -65,23 +64,10 @@ def _send(func_name, request):
     try:
         # FIXME POST?
         func(request.GET)
-    except (InfluxDBServerError, InfluxDBClientError) as e:
+    except IOError as e:
         logger.exception(e)
 
 
-def get_influxdb_client():
-    # TODO use UDP?
-    client = InfluxDBClient(
-        settings.INFLUXDB_HOST,
-        settings.INFLUXDB_PORT,
-        settings.INFLUXDB_USER,
-        settings.INFLUXDB_PASSWORD,
-        settings.INFLUXDB_DB
-    )
-
-    return client
-
-
 def search(data):
     platforms = data.get('platforms', '').split()
     cloud_platforms = data.get('cloud_platforms', '').split()
@@ -91,39 +77,20 @@ def search(data):
     if not any((platforms, cloud_platforms, tags, keywords)):
         return
 
-    search_data = [{
-        'measurement': 'search',
-        'tags': {
-            'platform': ','.join(platforms),
-            'cloud_platform': ','.join(cloud_platforms),
-            'tag': ','.join(tags),
-            'keyword': ','.join(keywords),
-        },
-        'fields': {
-            'value': 1
-        },
-    },]
-
     search_criteria = {
         'keyword': keywords,
         'platform': platforms,
         'cloud_platform': cloud_platforms,
         'tag': tags,
     }
-    search_criteria_data = []
+
+    settings.PROM_CNTR_SEARCH.labels(
+        keywords=','.join(keywords),
+        platforms=','.join(platforms),
+        cloud_platforms=','.join(cloud_platforms),
+        tags=','.join(tags)
+    ).inc()
+
     for criteria_type, criteria_values in search_criteria.items():
         for criteria_value in criteria_values:
-            search_criteria_data.append({
-                'measurement': 'search_criteria',
-                'tags': {
-                    'type': criteria_type,
-                    'cname': criteria_value
-                },
-                'fields': {
-                    'value': 1
-                }
-            })
-
-    client = get_influxdb_client()
-    client.write_points(search_data)
-    client.write_points(search_criteria_data, retention_policy='a_month')
+            settings.PROM_CNTR_SEARCH_CRITERIA.labels(ctype=criteria_type, cvalue=criteria_value).inc()

galaxy/settings/default.py

@@ -19,6 +19,7 @@
 import os
 
 import djcelery
+import prometheus_client
 
 
 djcelery.setup_loader()
@@ -43,6 +44,8 @@
 # Application definition
 # ---------------------------------------------------------
 
+PROMETHEUS_EXPORT_MIGRATIONS = False
+
 # TODO(cutwater): Review 3rd party apps usage
 INSTALLED_APPS = (
     # Django apps
@@ -65,6 +68,7 @@
 
     # 3rd part apps
     'bootstrapform',
+    'django_prometheus',
     'djcelery',
     'rest_framework',
     'rest_framework.authtoken',
@@ -293,11 +297,20 @@
 # =========================================================
 
 METRICS_ENABLED = False
-INFLUXDB_HOST = None
-INFLUXDB_PORT = None
-INFLUXDB_DB = None
-INFLUXDB_USER = None
-INFLUXDB_PASSWORD = None
+
+PROM_CNTR_SEARCH = prometheus_client.Counter(
+    'search',
+    '',
+    ['keywords', 'platforms', 'cloud_platforms', 'tags'],
+    registry=prometheus_client.REGISTRY
+)
+
+PROM_CNTR_SEARCH_CRITERIA = prometheus_client.Counter(
+    'search_criteria',
+    '',
+    ['ctype', 'cvalue'],
+    registry=prometheus_client.REGISTRY
+)
 
 # =========================================================
 # Logging
@@ -379,7 +392,7 @@
             'level': 'DEBUG',
             'propagate': True,
         },
-        'galaxy.common.metricsutils': {
+        'galaxy.common.metrics': {
             'handlers': ['console'],
             'level': 'DEBUG',
             'propagate': True,

galaxy/settings/development.py

@@ -28,7 +28,8 @@
 
 DEBUG = True
 
-ALLOWED_HOSTS = ['*']
+#ALLOWED_HOSTS = ['*']
+ALLOWED_HOSTS = ['localhost', '127.0.0.1', '172.18.0.5']  # see scripts/docker/dev/prometheus.yml
 
 # Application definition
 # ---------------------------------------------------------
@@ -84,6 +85,7 @@
     {'host': 'rabbitmq', 'port': 5672},
     {'host': 'influxdb', 'port': 8086},
     {'host': 'grafana', 'port': 3000},
+    {'host': 'prometheus', 'port': 9090},
 ]
 
 STATIC_ROOT = ''
@@ -93,8 +95,3 @@
 # =========================================================
 
 METRICS_ENABLED = True
-INFLUXDB_HOST = 'influxdb'
-INFLUXDB_PORT = 8086
-INFLUXDB_DB = 'galaxy'
-INFLUXDB_USER = 'galaxy'
-INFLUXDB_PASSWORD = 'galaxy'

galaxy/settings/production.py

@@ -38,11 +38,6 @@
 * GALAXY_RABBITMQ_PASSWORD
 * GALAXY_ADMIN_PATH
 * GALAXY_METRICS_ENABLED
-* GALAXY_INFLUXDB_HOST
-* GALAXY_INFLUXDB_PORT
-* GALAXY_INFLUXDB_DB
-* GALAXY_INFLUXDB_USER
-* GALAXY_INFLUXDB_PASSWORD
 """
 
 import os
@@ -214,11 +209,6 @@ def _set_logging():
 # =========================================================
 
 METRICS_ENABLED = True
-INFLUXDB_HOST = os.environ.get('GALAXY_INFLUXDB_HOST', '')
-INFLUXDB_PORT = int(os.environ.get('GALAXY_INFLUXDB_PORT', 8086))
-INFLUXDB_DB = os.environ.get('GALAXY_INFLUXDB_DB', 'galaxy')
-INFLUXDB_USER = os.environ.get('GALAXY_INFLUXDB_USER', 'galaxy')
-INFLUXDB_PASSWORD = os.environ.get('GALAXY_INFLUXDB_PASSWORD', '')
 
 # =========================================================
 # System Settings

requirements.txt

@@ -58,4 +58,5 @@ django-log-request-id
 jog
 
 # Web analytics/metrics
-influxdb
+prometheus_client
+django-prometheus

scripts/docker/dev/compose.yml

@@ -15,7 +15,6 @@ services:
     links:
       - postgres
       - rabbitmq
-      - influxdb
 
   postgres:
     image: centos/postgresql-95-centos7
@@ -52,3 +51,13 @@ services:
       - '3000:3000'
     links:
       - influxdb
+      - prometheus
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - '9090:9090'
+    volumes:
+      - './prometheus.yml:/etc/prometheus/prometheus.yml'
+    links:
+      - galaxy

scripts/docker/dev/prometheus.yml

@@ -0,0 +1,12 @@
+scrape_configs:
+  - job_name: 'galaxy'
+    scrape_interval: '60s'
+    honor_labels: true
+    metrics_path: '/api/metrics'
+    static_configs:
+      - targets:
+          - '172.18.0.5:8000'  # FIXME does not work with 'galaxy:8000' - "Invalid Host Header"
+          # - another galaxy instance
+
+remote_write:
+  - url: "http://influxdb:8086/api/v1/prom/write?u=galaxy&p=galaxy&db=galaxy"

scripts/metrics-setup-playbook/influxdb_init.py

@@ -19,26 +19,26 @@
 client.switch_database(INFLUXDB_DB)
 
 # Retention policies
-client.query(
-    "CREATE RETENTION POLICY a_month ON %s DURATION 30d REPLICATION 1"
-    % INFLUXDB_DB
-)
+#client.query(
+#    "CREATE RETENTION POLICY a_month ON %s DURATION 30d REPLICATION 1"
+#    % INFLUXDB_DB
+#)
 client.query(
     "CREATE RETENTION POLICY a_year ON %s DURATION 365d REPLICATION 1 DEFAULT"
     % INFLUXDB_DB
 )
 
 # Continuous queries
-client.query(
-    "DROP CONTINUOUS QUERY cq_search_criteria_per_day ON %s" % INFLUXDB_DB
-)
-
-client.query(
-    "CREATE CONTINUOUS QUERY cq_search_criteria_per_day ON %s "
-    "RESAMPLE EVERY 30m "
-    "BEGIN "
-    "SELECT count(value) AS value INTO search_criteria_per_day FROM a_month.search_criteria "
-    "GROUP BY time(1d), * "
-    "END"
-    % INFLUXDB_DB
-)
+#client.query(
+#    "DROP CONTINUOUS QUERY cq_search_criteria_per_day ON %s" % INFLUXDB_DB
+#)
+#
+#client.query(
+#    "CREATE CONTINUOUS QUERY cq_search_criteria_per_day ON %s "
+#    "RESAMPLE EVERY 30m "
+#    "BEGIN "
+#    "SELECT count(value) AS value INTO search_criteria_per_day FROM a_month.search_criteria "
+#    "GROUP BY time(1d), * "
+#    "END"
+#    % INFLUXDB_DB
+#)