Stop using context manager for DataFetcher.start, as it wasn't necess…

…ary and added complexity.

Hack support for expanding the short, internal source name into a longer "query" source name.

tests/test_query_rawr.py

@@ -1,5 +1,4 @@
 import unittest
-from contextlib import contextmanager
 
 
 class TestGetTable(object):
@@ -21,9 +20,8 @@ class ConstantStorage(object):
     def __init__(self, tables):
         self.tables = tables
 
-    @contextmanager
     def __call__(self, top_tile):
-        yield self.tables
+        return self.tables
 
 
 class RawrTestCase(unittest.TestCase):
@@ -254,6 +252,53 @@ def _rel(id, nodes=None, ways=None, rels=None):
             _rel(5, rels=[2, 4]),
         ], 5)
 
+    def test_query_source(self):
+        # check that the source is added to the result properties, and that
+        # it overrides any existing source.
+
+        from shapely.geometry import Point
+        from tilequeue.query.rawr import TilePyramid
+        from tilequeue.tile import coord_to_mercator_bounds
+        from tilequeue.tile import mercator_point_to_coord
+
+        feature_min_zoom = 11
+
+        def min_zoom_fn(shape, props, fid, meta):
+            return feature_min_zoom
+
+        shape = Point(0, 0)
+        # get_table(table_name) should return a generator of rows.
+        tables = TestGetTable({
+            'planet_osm_point': [(0, shape.wkb,
+                                  {'source': 'originalrowsource'})],
+        })
+
+        zoom = 10
+        max_zoom = zoom + 5
+        coord = mercator_point_to_coord(zoom, shape.x, shape.y)
+        tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
+
+        fetch = self._make(min_zoom_fn, None, tables, tile_pyramid,
+                           source='testingquerysource')
+
+        # first, check that it can get the original item back when both the
+        # min zoom filter and geometry filter are okay.
+        feature_coord = mercator_point_to_coord(
+            feature_min_zoom, shape.x, shape.y)
+        for fetcher, _ in fetch.start(_wrap(coord)):
+            read_rows = fetcher(
+                feature_min_zoom, coord_to_mercator_bounds(feature_coord))
+
+        self.assertEquals(1, len(read_rows))
+        read_row = read_rows[0]
+        self.assertEquals(0, read_row.get('__id__'))
+        # query processing code expects WKB bytes in the __geometry__ column
+        self.assertEquals(shape.wkb, read_row.get('__geometry__'))
+        self.assertEquals({'min_zoom': 11},
+                          read_row.get('__testlayer_properties__'))
+        self.assertEquals({'source': 'testingquerysource'},
+                          read_row.get('__properties__'))
+
 
 class TestLabelPlacement(RawrTestCase):
 

tilequeue/command.py

@@ -1455,7 +1455,8 @@ def tilequeue_process_tile(cfg, peripherals, args):
     data_fetcher = make_data_fetcher(cfg, layer_data, query_cfg, io_pool)
 
     unpadded_bounds = coord_to_mercator_bounds(coord)
-    source_rows = data_fetcher(coord.zoom, unpadded_bounds)
+    for fetch, _ in data_fetcher.start([dict(coord=coord)]):
+        source_rows = fetch(coord.zoom, unpadded_bounds)
     feature_layers = convert_source_data_to_feature_layers(
         source_rows, layer_data, unpadded_bounds, coord.zoom)
     cut_coords = []

tilequeue/query/__init__.py

@@ -124,24 +124,35 @@ def _make_layer_info(layer_data, process_yaml_cfg):
 
 
 def _parse_shape_types(inputs):
-    from tilequeue.query.common import shape_type_lookup
-
     outputs = set()
     for value in inputs:
-        outputs.add(shape_type_lookup(value))
+        if value.startswith('Multi'):
+            value = value[len('Multi'):]
+        outputs.add(value.lower())
 
     if outputs:
         return outputs
     else:
         return None
 
 
+def _parse_layers(yaml_path, output_fn, make_function_name_fn):
+    from vectordatasource.meta.python import parse_layers
+
+    layer_parse_result = parse_layers(
+        yaml_path, output_fn, make_function_name_fn)
+
+    layers = {}
+    for function_data in layer_parse_result.layer_data:
+        layers[function_data.layer] = function_data.fn
+    return layers
+
+
 def _parse_yaml_functions(process_yaml_cfg):
     from vectordatasource.meta.python import make_function_name_props
     from vectordatasource.meta.python import make_function_name_min_zoom
     from vectordatasource.meta.python import output_kind
     from vectordatasource.meta.python import output_min_zoom
-    from vectordatasource.meta.python import parse_layers
     import os.path
 
     # can't handle "callable" type - how do we get the min zoom fn?
@@ -152,9 +163,9 @@ def _parse_yaml_functions(process_yaml_cfg):
 
     assert os.path.isdir(yaml_path)
 
-    output_layer_data = parse_layers(
+    output_layer_data = _parse_layers(
         yaml_path, output_kind, make_function_name_props)
-    min_zoom_layer_data = parse_layers(
+    min_zoom_layer_data = _parse_layers(
         yaml_path, output_min_zoom, make_function_name_min_zoom)
 
     keys = set(output_layer_data.keys())

tilequeue/query/common.py

@@ -256,6 +256,11 @@ def mz_calculate_transit_routes_and_score(osm, node_id, way_id):
 def layer_properties(fid, shape, props, layer_name, zoom, osm):
     layer_props = props.copy()
 
+    # drop the 'source' tag, if it exists. we override it anyway, and it just
+    # gets confusing having multiple source tags. in the future, we may
+    # replace the whole thing with a separate 'meta' for source.
+    layer_props.pop('source', None)
+
     # need to make sure that the name is only applied to one of
     # the pois, landuse or buildings layers - in that order of
     # priority.

tilequeue/query/rawr.py

@@ -173,8 +173,11 @@ def add_feature(self, fid, shape_wkb, props):
     def add_way(self, way_id, nodes, tags):
         for node_id in nodes:
             if node_id in self.nodes:
-                assert way_id in self.ways
-                self._ways_using_node[node_id].append(way_id)
+                # a way might be missing here if we filtered it out because it
+                # was not interesting, e.g: not a road, station, etc... that
+                # we might need to look up later.
+                if way_id in self.ways:
+                    self._ways_using_node[node_id].append(way_id)
 
     def add_relation(self, rel_id, way_off, rel_off, parts, members, tags):
         r = Relation(rel_id, way_off, rel_off, parts, members, tags)
@@ -324,6 +327,17 @@ def _make_meta(source, fid, shape_type, osm):
     return _Metadata(source, ways, rel_dicts)
 
 
+_EXPANDED_SOURCES = {
+    'osm': 'openstreetmap.org',
+}
+
+
+# horrible little hack, as in some places we use source="osm" and in other
+# places use source="openstreetmap.org", and so on for other sources.
+def _expand_source(source):
+    return _EXPANDED_SOURCES.get(source, source)
+
+
 class _LayersIndex(object):
     """
     Index features by the tile(s) that they appear in.
@@ -557,6 +571,10 @@ def __call__(self, zoom, unpadded_bounds):
                     read_row['__label__'] = bytes(
                         shape.representative_point().wkb)
 
+                if self.source:
+                    source = _expand_source(self.source)
+                    read_row['__properties__'] = {'source': source}
+
                 read_rows.append(read_row)
 
         return read_rows
@@ -589,12 +607,13 @@ def start(self, all_data):
                 self.min_z, int(top_coord.column), int(top_coord.row),
                 self.max_z)
 
-            with self.storage(tile_pyramid) as tables:
-                fetcher = RawrTile(self.layers, tables, tile_pyramid,
-                                   self.label_placement_layers, self.source)
+            tables = self.storage(tile_pyramid.tile())
+
+            fetcher = RawrTile(self.layers, tables, tile_pyramid,
+                               self.label_placement_layers, self.source)
 
-                for coord, data in coord_group:
-                    yield fetcher, data
+            for coord, data in coord_group:
+                yield fetcher, data
 
 
 # Make a RAWR tile data fetcher given: