Merge pull request #155 from tilezen/zerebubuth/remove-layers-to-format

Remove "layers to format" functionality.

config.yaml.sample

@@ -98,13 +98,6 @@ process:
   # extensions of formats to generate
   # buffered Mapbox Vector Tiles are also possible by specifying mvtb
   formats: [json, topojson, mvt]
-  # in addition to the `all` layer, any additional layers to also
-  # store during processing
-  layers-to-format:
-    - layer: buildings
-      formats: [json]
-      zoom-start: 13
-      zoom-until: 20
   # additionally, the data included for some formats expects to be
   # buffered. This is where buffers per layer or per geometry type can
   # be specified, with layers trumping geometry types

tilequeue/command.py

@@ -563,7 +563,7 @@ def tilequeue_process(cfg, peripherals):
 
     data_processor = ProcessAndFormatData(
         post_process_data, formats, sql_data_fetch_queue, processor_queue,
-        cfg.layers_to_format, cfg.buffer_cfg, logger)
+        cfg.buffer_cfg, logger)
 
     s3_storage = S3Storage(processor_queue, s3_store_queue, io_pool,
                            store, logger, cfg.metatile_size, cfg.store_orig)

tilequeue/config.py

@@ -86,19 +86,6 @@ def __init__(self, yml):
         self.output_formats = process_cfg['formats']
         self.buffer_cfg = process_cfg['buffer']
 
-        layers_to_format_cfg = process_cfg.get('layers-to-format')
-        layers_to_format = []
-        from tilequeue.command import lookup_formats
-        for layer_to_format_cfg in layers_to_format_cfg:
-            layer_name = layer_to_format_cfg['layer']
-            format_names = layer_to_format_cfg['formats']
-            formats = lookup_formats(format_names)
-            zoom_start = int(layer_to_format_cfg.get('zoom-start', 0))
-            zoom_until = int(layer_to_format_cfg.get('zoom-until', 20))
-            layer_to_format = layer_name, formats, zoom_start, zoom_until
-            layers_to_format.append(layer_to_format)
-        self.layers_to_format = layers_to_format
-
         self.postgresql_conn_info = self.yml['postgresql']
         dbnames = self.postgresql_conn_info.get('dbnames')
         assert dbnames is not None, 'Missing postgresql dbnames'
@@ -179,7 +166,6 @@ def default_yml_config():
             'template-path': None,
             'reload-templates': False,
             'formats': ['json'],
-            'layers-to-format': [],
             'buffer': {},
         },
         'logging': {

tilequeue/process.py

@@ -268,7 +268,7 @@ def _create_formatted_tile(
 
 def _process_feature_layers(
         feature_layers, coord, post_process_data, formats, unpadded_bounds,
-        scale, layers_to_format, buffer_cfg):
+        scale, buffer_cfg):
 
     # the nominal zoom is the "display scale" zoom, which may not correspond
     # to actual tile coordinates in future versions of the code. it just
@@ -338,30 +338,14 @@ def _process_feature_layers(
             buffer_cfg)
         formatted_tiles.append(formatted_tile)
 
-    # this assumes that we only store single layers, and no combinations
-    for layer, formats, zoom_start, zoom_until in layers_to_format:
-        if not (zoom_start <= coord.zoom <= zoom_until):
-            continue
-        for feature_layer in processed_feature_layers:
-            if feature_layer['name'] == layer:
-                pruned_feature_layers = [feature_layer]
-                for format in formats:
-                    formatted_tile = _create_formatted_tile(
-                        pruned_feature_layers, format, scale, unpadded_bounds,
-                        unpadded_bounds_lnglat, coord, layer,
-                        meters_per_pixel_dim, buffer_cfg)
-                    formatted_tiles.append(formatted_tile)
-                    break
-
     return formatted_tiles
 
 
 # given a coord and the raw feature layers results from the database,
 # filter, transform, sort, post-process and then format according to
 # each formatter. this is the entry point from the worker process
 def process_coord(coord, feature_layers, post_process_data, formats,
-                  unpadded_bounds, cut_coords, layers_to_format,
-                  buffer_cfg, scale=4096):
+                  unpadded_bounds, cut_coords, buffer_cfg, scale=4096):
     feature_layers, extra_data = _preprocess_data(feature_layers)
 
     children_formatted_tiles = []
@@ -375,11 +359,11 @@ def process_coord(coord, feature_layers, post_process_data, formats,
                 buffer_cfg)
             child_formatted_tiles = _process_feature_layers(
                 child_feature_layers, cut_coord, post_process_data, formats,
-                unpadded_cut_bounds, scale, layers_to_format, buffer_cfg)
+                unpadded_cut_bounds, scale, buffer_cfg)
             children_formatted_tiles.extend(child_formatted_tiles)
 
     coord_formatted_tiles = _process_feature_layers(
         feature_layers, coord, post_process_data, formats, unpadded_bounds,
-        scale, layers_to_format, buffer_cfg)
+        scale, buffer_cfg)
     all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
     return all_formatted_tiles, extra_data

tilequeue/worker.py

@@ -200,13 +200,12 @@ class ProcessAndFormatData(object):
     scale = 4096
 
     def __init__(self, post_process_data, formats, input_queue,
-                 output_queue, layers_to_format, buffer_cfg, logger):
+                 output_queue, buffer_cfg, logger):
         formats.sort(key=attrgetter('sort_key'))
         self.post_process_data = post_process_data
         self.formats = formats
         self.input_queue = input_queue
         self.output_queue = output_queue
-        self.layers_to_format = layers_to_format
         self.buffer_cfg = buffer_cfg
         self.logger = logger
 
@@ -235,7 +234,7 @@ def __call__(self, stop):
                 formatted_tiles, extra_data = process_coord(
                     coord, feature_layers, self.post_process_data,
                     self.formats, unpadded_bounds, cut_coords,
-                    self.layers_to_format, self.buffer_cfg)
+                    self.buffer_cfg)
             except:
                 stacktrace = format_stacktrace_one_line()
                 self.logger.error('Error processing: %s - %s' % (