Merge pull request #281 from tilezen/zerebubuth/various-rawr-tile-fixes

Various RAWR tile fixes

tests/test_query_common.py

@@ -0,0 +1,45 @@
+import unittest
+
+
+class TestCommon(unittest.TestCase):
+
+    def test_parse_shape_types(self):
+        from tilequeue.query.common import ShapeType
+
+        def _test(expects, inputs):
+            self.assertEquals(set(expects), ShapeType.parse_set(inputs))
+
+        # basic types
+        _test([ShapeType.point], ['point'])
+        _test([ShapeType.line], ['line'])
+        _test([ShapeType.polygon], ['polygon'])
+
+        # should be case insensitive
+        _test([ShapeType.point], ['Point'])
+        _test([ShapeType.line], ['LINE'])
+        _test([ShapeType.polygon], ['Polygon'])
+
+        # should handle OGC-style names, including Multi*
+        _test([ShapeType.point], ['MultiPoint'])
+        _test([ShapeType.line], ['LineString'])
+        _test([ShapeType.line], ['MultiLineString'])
+        _test([ShapeType.polygon], ['MultiPolygon'])
+
+        # should handle multiple, repeated names
+        _test([ShapeType.point], ['Point', 'MultiPoint'])
+        _test([
+            ShapeType.point,
+            ShapeType.line,
+            ShapeType.polygon,
+        ], [
+            'Point', 'MultiPoint',
+            'Line', 'LineString', 'MultiLineString',
+            'Polygon', 'MultiPolygon'
+        ])
+
+        # should return None rather than an empty set.
+        self.assertEquals(None, ShapeType.parse_set([]))
+
+        # should throw KeyError if the name isn't recognised
+        with self.assertRaises(KeyError):
+            ShapeType.parse_set(['MegaPolygon'])

tests/test_query_rawr.py

@@ -671,3 +671,63 @@ def min_zoom_fn(shape, props, fid, meta):
         self.assertEquals(1, read_row.get('__id__'))
         self.assertEquals({'min_zoom': 11, 'highway': 'secondary'},
                           read_row.get('__testlayer_properties__'))
+
+
+class TestTileFootprint(unittest.TestCase):
+
+    def test_single_tile(self):
+        from tilequeue.query.rawr import _tiles
+        from raw_tiles.tile import Tile
+        from raw_tiles.util import bbox_for_tile
+
+        tile = Tile(15, 5241, 12665)
+
+        zoom = tile.z
+        unpadded_bounds = bbox_for_tile(tile.z, tile.x, tile.y)
+        tiles = _tiles(zoom, unpadded_bounds)
+
+        self.assertEquals([tile], list(tiles))
+
+    def test_multiple_tiles(self):
+        from tilequeue.query.rawr import _tiles
+        from raw_tiles.tile import Tile
+        from raw_tiles.util import bbox_for_tile
+
+        tile = Tile(15, 5241, 12665)
+
+        # query at one zoom higher - should get 4 child tiles.
+        zoom = tile.z + 1
+        unpadded_bounds = bbox_for_tile(tile.z, tile.x, tile.y)
+        tiles = list(_tiles(zoom, unpadded_bounds))
+
+        self.assertEquals(4, len(tiles))
+        for child in tiles:
+            self.assertEquals(tile, child.parent())
+
+    def test_corner_overlap(self):
+        # a box around the corner of a tile should return the four
+        # neighbours of that tile.
+        from tilequeue.query.rawr import _tiles
+        from raw_tiles.tile import Tile
+        from raw_tiles.util import bbox_for_tile
+
+        tile = Tile(15, 5241, 12665)
+
+        zoom = tile.z
+        tile_bbox = bbox_for_tile(tile.z, tile.x, tile.y)
+
+        # extract the top left corner
+        x = tile_bbox[0]
+        y = tile_bbox[3]
+        # make a small bounding box around that
+        w = 10
+        unpadded_bounds = (x - w, y - w,
+                           x + w, y + w)
+        tiles = set(_tiles(zoom, unpadded_bounds))
+
+        expected = set()
+        for dx in (0, -1):
+            for dy in (0, -1):
+                expected.add(Tile(tile.z, tile.x + dx, tile.y + dy))
+
+        self.assertEquals(expected, tiles)

tilequeue/query/__init__.py

@@ -141,31 +141,21 @@ def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool):
 
 
 def _make_layer_info(layer_data, process_yaml_cfg):
-    from tilequeue.query.common import LayerInfo
+    from tilequeue.query.common import LayerInfo, ShapeType
 
     layers = {}
     functions = _parse_yaml_functions(process_yaml_cfg)
 
     for layer_datum in layer_data:
         name = layer_datum['name']
         min_zoom_fn, props_fn = functions[name]
-        shape_types = _parse_shape_types(layer_datum['geometry_types'])
+        shape_types = ShapeType.parse_set(layer_datum['geometry_types'])
         layer_info = LayerInfo(min_zoom_fn, props_fn, shape_types)
         layers[name] = layer_info
 
     return layers
 
 
-def _parse_shape_types(inputs):
-    outputs = set()
-    for value in inputs:
-        if value.startswith('Multi'):
-            value = value[len('Multi'):]
-        outputs.add(value.lower())
-
-    return outputs or None
-
-
 def _parse_yaml_functions(process_yaml_cfg):
     from tilequeue.command import make_output_calc_mapping
     from tilequeue.command import make_min_zoom_calc_mapping

tilequeue/query/common.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 from itertools import izip
 from tilequeue.process import Source
+from enum import Enum
 
 
 def namedtuple_with_defaults(name, props, defaults):
@@ -20,6 +21,45 @@ def allows_shape_type(self, shape):
         return typ in self.shape_types
 
 
+class ShapeType(Enum):
+    point = 1
+    line = 2
+    polygon = 3
+
+    # aliases, don't use these directly!
+    multipoint = 1
+    linestring = 2
+    multilinestring = 2
+    multipolygon = 3
+
+    @classmethod
+    def parse_set(cls, inputs):
+        outputs = set()
+        for value in inputs:
+            t = cls[value.lower()]
+            outputs.add(t)
+
+        return outputs or None
+
+
+# determine the shape type from the raw WKB bytes. this means we don't have to
+# parse the WKB, which can be an expensive operation for large polygons.
+def wkb_shape_type(wkb):
+    reverse = ord(wkb[0]) == 1
+    type_bytes = map(ord, wkb[1:5])
+    if reverse:
+        type_bytes.reverse()
+    typ = type_bytes[3]
+    if typ == 1 or typ == 4:
+        return ShapeType.point
+    elif typ == 2 or typ == 5:
+        return ShapeType.line
+    elif typ == 3 or typ == 6:
+        return ShapeType.polygon
+    else:
+        assert False, "WKB shape type %d not understood." % (typ,)
+
+
 def deassoc(x):
     """
     Turns an array consisting of alternating key-value pairs into a
@@ -176,7 +216,7 @@ def mz_calculate_transit_routes_and_score(osm, node_id, way_id, rel_id):
     seed_relations = set()
     for rel_id in candidate_relations:
         rel = osm.relation(rel_id)
-        if mz_is_interesting_transit_relation(rel.tags):
+        if rel and mz_is_interesting_transit_relation(rel.tags):
             seed_relations.add(rel_id)
     del candidate_relations
 
@@ -202,8 +242,11 @@ def mz_calculate_transit_routes_and_score(osm, node_id, way_id, rel_id):
     stations_and_stops = set()
     for rel_id in all_relations:
         rel = osm.relation(rel_id)
+        if not rel:
+            continue
         for node_id in rel.node_ids:
-            if is_station_or_stop(*osm.node(node_id)):
+            node = osm.node(node_id)
+            if node and is_station_or_stop(*node):
                 stations_and_stops.add(node_id)
 
     if node_id:
@@ -214,7 +257,8 @@ def mz_calculate_transit_routes_and_score(osm, node_id, way_id, rel_id):
     stations_and_lines = set()
     for node_id in stations_and_stops:
         for way_id in osm.ways_using_node(node_id):
-            if is_station_or_line(*osm.way(way_id)):
+            way = osm.way(way_id)
+            if way and is_station_or_line(*way):
                 stations_and_lines.add(way_id)
 
     if way_id:
@@ -229,23 +273,26 @@ def mz_calculate_transit_routes_and_score(osm, node_id, way_id, rel_id):
         for i in ids:
             for rel_id in lookup(i):
                 rel = osm.relation(rel_id)
-                if rel.tags.get('type') == 'route' and \
+                if rel and \
+                   rel.tags.get('type') == 'route' and \
                    rel.tags.get('route') in ('subway', 'light_rail', 'tram',
                                              'train', 'railway'):
                     all_routes.add(rel_id)
 
     routes_lookup = defaultdict(set)
     for rel_id in all_routes:
         rel = osm.relation(rel_id)
+        if not rel:
+            continue
         route = rel.tags.get('route')
         if route:
             route_name = mz_transit_route_name(rel.tags)
             routes_lookup[route].add(route_name)
-    trains = routes_lookup['train']
-    subways = routes_lookup['subway']
-    light_rails = routes_lookup['light_rail']
-    trams = routes_lookup['tram']
-    railways = routes_lookup['railway']
+    trains = list(sorted(routes_lookup['train']))
+    subways = list(sorted(routes_lookup['subway']))
+    light_rails = list(sorted(routes_lookup['light_rail']))
+    trams = list(sorted(routes_lookup['tram']))
+    railways = list(sorted(routes_lookup['railway']))
     del routes_lookup
 
     # if a station is an interchange between mainline rail and subway or
@@ -336,6 +383,8 @@ def layer_properties(fid, shape, props, layer_name, zoom, osm):
         mz_is_bus_route = False
         for rel_id in osm.relations_using_way(fid):
             rel = osm.relation(rel_id)
+            if not rel:
+                continue
             typ, route, network, ref = [rel.tags.get(k) for k in (
                 'type', 'route', 'network', 'ref')]
             if route and (network or ref):