Add statsd support to tilequeue

config.yaml.sample

@@ -8,6 +8,11 @@ queue:
     # values are the queue name to use for zooms in that range
     0-10: queue-1
     11-20: queue-2
+statsd:
+  # Uncomment host to activate statsd
+  # host: 127.0.0.1
+  port: 8125
+  prefix: dev.tilequeue
 store:
   type: s3 # Can also be `directory`, which would dump the tiles to disk.
   name: <s3 bucket/tile directory name>

requirements.txt

@@ -18,6 +18,7 @@ requests==2.10.0
 Shapely==1.4.3
 six==1.10.0
 StreetNames==0.1.5
+statsd==3.2.1
 ujson==1.35
 Werkzeug==0.9.6
 wsgiref==0.1.2

setup.py

@@ -47,6 +47,7 @@
           'redis',
           'requests',
           'Shapely',
+          'statsd',
           'ujson',
           'zope.dottedname',
       ],

tilequeue/command.py

@@ -1076,6 +1076,7 @@ def tilequeue_prune_tiles_of_interest(cfg, peripherals):
     toi_to_remove = tiles_of_interest - new_toi
     logger.info('Computing tiles to remove ... done. %s found',
                 len(toi_to_remove))
+    peripherals.stats.gauge('gardener.removed', len(toi_to_remove))
 
     def delete_tile_of_interest(s3_parts, coord_ints):
         # Remove from the redis toi set
@@ -1119,6 +1120,7 @@ def delete_tile_of_interest(s3_parts, coord_ints):
     toi_to_add = new_toi - tiles_of_interest
     logger.info('Computing tiles to add ... done. %s found',
                 len(toi_to_add))
+    peripherals.stats.gauge('gardener.added', len(toi_to_remove))
 
     if not toi_to_add:
         logger.info('Skipping TOI add step because there are '
@@ -1496,6 +1498,26 @@ def error(self, message):
         sys.exit(2)
 
 
+class FakeStatsd(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def incr(self, *args, **kwargs):
+        pass
+
+    def decr(self, *args, **kwargs):
+        pass
+
+    def gauge(self, *args, **kwargs):
+        pass
+
+    def set(self, *args, **kwargs):
+        pass
+
+    def timing(self, *args, **kwargs):
+        pass
+
+
 def tilequeue_main(argv_args=None):
     if argv_args is None:
         argv_args = sys.argv[1:]
@@ -1531,10 +1553,17 @@ def tilequeue_main(argv_args=None):
         'Config file {} does not exist!'.format(args.config)
     cfg = make_config_from_argparse(args.config)
     redis_client = make_redis_client(cfg)
-    Peripherals = namedtuple('Peripherals', 'redis_cache_index queue')
+    Peripherals = namedtuple('Peripherals', 'redis_cache_index queue stats')
     queue = make_queue(
         cfg.queue_type, cfg.queue_name, cfg.queue_cfg, redis_client,
         aws_access_key_id=cfg.aws_access_key_id,
         aws_secret_access_key=cfg.aws_secret_access_key)
-    peripherals = Peripherals(make_redis_cache_index(redis_client, cfg), queue)
+    if cfg.statsd_host:
+        import statsd
+        stats = statsd.StatsClient(cfg.statsd_host, cfg.statsd_port,
+                                   prefix=cfg.statsd_prefix)
+    else:
+        stats = FakeStatsd()
+    peripherals = Peripherals(make_redis_cache_index(redis_client, cfg), queue,
+                              stats)
     args.func(cfg, peripherals)

tilequeue/config.py

@@ -72,6 +72,12 @@ def __init__(self, yml):
         self.redis_db = self._cfg('redis db')
         self.redis_cache_set_key = self._cfg('redis cache-set-key')
 
+        self.statsd_host = None
+        if self.yml.get('statsd'):
+            self.statsd_host = self._cfg('statsd host')
+            self.statsd_port = self._cfg('statsd port')
+            self.statsd_prefix = self._cfg('statsd prefix')
+
         process_cfg = self.yml['process']
         self.n_simultaneous_query_sets = \
             process_cfg['n-simultaneous-query-sets']