Remove "layers to format" functionality.

This was used to generate extracts of specific layers to be stored in S3 so that they could be served directly for single layer requests. However, we now use a microservice called "xonacatl" to serve dynamic layers requests and so this functionality isn't needed any more.

config.yaml.sample

@@ -98,13 +98,6 @@ process:
   # extensions of formats to generate
   # buffered Mapbox Vector Tiles are also possible by specifying mvtb
   formats: [json, topojson, mvt]
-  # in addition to the `all` layer, any additional layers to also
-  # store during processing
-  layers-to-format:
-    - layer: buildings
-      formats: [json]
-      zoom-start: 13
-      zoom-until: 20
   # additionally, the data included for some formats expects to be
   # buffered. This is where buffers per layer or per geometry type can
   # be specified, with layers trumping geometry types

tilequeue/command.py

@@ -563,7 +563,7 @@ def tilequeue_process(cfg, peripherals):
 
     data_processor = ProcessAndFormatData(
         post_process_data, formats, sql_data_fetch_queue, processor_queue,
-        cfg.layers_to_format, cfg.buffer_cfg, logger)
+        cfg.buffer_cfg, logger)
 
     s3_storage = S3Storage(processor_queue, s3_store_queue, io_pool,
                            store, logger, cfg.metatile_size, cfg.store_orig)

tilequeue/config.py

@@ -86,19 +86,6 @@ def __init__(self, yml):
         self.output_formats = process_cfg['formats']
         self.buffer_cfg = process_cfg['buffer']
 
-        layers_to_format_cfg = process_cfg.get('layers-to-format')
-        layers_to_format = []
-        from tilequeue.command import lookup_formats
-        for layer_to_format_cfg in layers_to_format_cfg:
-            layer_name = layer_to_format_cfg['layer']
-            format_names = layer_to_format_cfg['formats']
-            formats = lookup_formats(format_names)
-            zoom_start = int(layer_to_format_cfg.get('zoom-start', 0))
-            zoom_until = int(layer_to_format_cfg.get('zoom-until', 20))
-            layer_to_format = layer_name, formats, zoom_start, zoom_until
-            layers_to_format.append(layer_to_format)
-        self.layers_to_format = layers_to_format
-
         self.postgresql_conn_info = self.yml['postgresql']
         dbnames = self.postgresql_conn_info.get('dbnames')
         assert dbnames is not None, 'Missing postgresql dbnames'
@@ -179,7 +166,6 @@ def default_yml_config():
             'template-path': None,
             'reload-templates': False,
             'formats': ['json'],
-            'layers-to-format': [],
             'buffer': {},
         },
         'logging': {

tilequeue/process.py

@@ -268,7 +268,7 @@ def _create_formatted_tile(
 
 def _process_feature_layers(
         feature_layers, coord, post_process_data, formats, unpadded_bounds,
-        scale, layers_to_format, buffer_cfg):
+        scale, buffer_cfg):
 
     processed_feature_layers = []
     # filter, and then transform each layer as necessary
@@ -331,30 +331,14 @@ def _process_feature_layers(
             buffer_cfg)
         formatted_tiles.append(formatted_tile)
 
-    # this assumes that we only store single layers, and no combinations
-    for layer, formats, zoom_start, zoom_until in layers_to_format:
-        if not (zoom_start <= coord.zoom <= zoom_until):
-            continue
-        for feature_layer in processed_feature_layers:
-            if feature_layer['name'] == layer:
-                pruned_feature_layers = [feature_layer]
-                for format in formats:
-                    formatted_tile = _create_formatted_tile(
-                        pruned_feature_layers, format, scale, unpadded_bounds,
-                        unpadded_bounds_lnglat, coord, layer,
-                        meters_per_pixel_dim, buffer_cfg)
-                    formatted_tiles.append(formatted_tile)
-                    break
-
     return formatted_tiles
 
 
 # given a coord and the raw feature layers results from the database,
 # filter, transform, sort, post-process and then format according to
 # each formatter. this is the entry point from the worker process
 def process_coord(coord, feature_layers, post_process_data, formats,
-                  unpadded_bounds, cut_coords, layers_to_format,
-                  buffer_cfg, scale=4096):
+                  unpadded_bounds, cut_coords, buffer_cfg, scale=4096):
     feature_layers, extra_data = _preprocess_data(feature_layers)
 
     children_formatted_tiles = []
@@ -368,11 +352,11 @@ def process_coord(coord, feature_layers, post_process_data, formats,
                 buffer_cfg)
             child_formatted_tiles = _process_feature_layers(
                 child_feature_layers, cut_coord, post_process_data, formats,
-                unpadded_cut_bounds, scale, layers_to_format, buffer_cfg)
+                unpadded_cut_bounds, scale, buffer_cfg)
             children_formatted_tiles.extend(child_formatted_tiles)
 
     coord_formatted_tiles = _process_feature_layers(
         feature_layers, coord, post_process_data, formats, unpadded_bounds,
-        scale, layers_to_format, buffer_cfg)
+        scale, buffer_cfg)
     all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
     return all_formatted_tiles, extra_data

tilequeue/worker.py

@@ -200,13 +200,12 @@ class ProcessAndFormatData(object):
     scale = 4096
 
     def __init__(self, post_process_data, formats, input_queue,
-                 output_queue, layers_to_format, buffer_cfg, logger):
+                 output_queue, buffer_cfg, logger):
         formats.sort(key=attrgetter('sort_key'))
         self.post_process_data = post_process_data
         self.formats = formats
         self.input_queue = input_queue
         self.output_queue = output_queue
-        self.layers_to_format = layers_to_format
         self.buffer_cfg = buffer_cfg
         self.logger = logger
 
@@ -235,7 +234,7 @@ def __call__(self, stop):
                 formatted_tiles, extra_data = process_coord(
                     coord, feature_layers, self.post_process_data,
                     self.formats, unpadded_bounds, cut_coords,
-                    self.layers_to_format, self.buffer_cfg)
+                    self.buffer_cfg)
             except:
                 stacktrace = format_stacktrace_one_line()
                 self.logger.error('Error processing: %s - %s' % (