Merge pull request #550 from mapzen/sort-key-csv-prototype

Sort key CSV prototype

queries.yaml

@@ -60,7 +60,6 @@ layers:
       - TileStache.Goodies.VecTiles.transform.tags_remove
       - TileStache.Goodies.VecTiles.transform.remap_deprecated_landuse_kinds
       - TileStache.Goodies.VecTiles.transform.normalize_tourism_kind
-      - TileStache.Goodies.VecTiles.transform.landuse_sort_key
       - TileStache.Goodies.VecTiles.transform.add_id_to_properties
       - TileStache.Goodies.VecTiles.transform.detect_osm_relation
       - TileStache.Goodies.VecTiles.transform.remove_feature_id
@@ -79,7 +78,6 @@ layers:
       - TileStache.Goodies.VecTiles.transform.parse_layer_as_float
       - TileStache.Goodies.VecTiles.transform.road_kind
       - TileStache.Goodies.VecTiles.transform.road_classifier
-      - TileStache.Goodies.VecTiles.transform.road_sort_key
       - TileStache.Goodies.VecTiles.transform.road_oneway
       - TileStache.Goodies.VecTiles.transform.road_abbreviate_name
       - TileStache.Goodies.VecTiles.transform.route_name
@@ -157,6 +155,73 @@ layers:
       - TileStache.Goodies.VecTiles.transform.remove_feature_id
     sort: TileStache.Goodies.VecTiles.sort.transit
 post_process:
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/landuse.csv
+    params:
+      source_layer: landuse
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/roads.csv
+    params:
+      source_layer: roads
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/earth.csv
+    params:
+      source_layer: earth
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/boundaries.csv
+    params:
+      source_layer: boundaries
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/buildings.csv
+    params:
+      source_layer: buildings
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/transit.csv
+    params:
+      source_layer: transit
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.csv_match_properties
+    resources:
+      matcher:
+        type: file
+        init_fn: TileStache.Goodies.VecTiles.transform.CSVMatcher
+        path: spreadsheets/water.csv
+    params:
+      source_layer: water
+      target_value_type: int
+  - fn: TileStache.Goodies.VecTiles.transform.drop_properties
+    params:
+      source_layer: roads
+      properties: [layer]
   - fn: TileStache.Goodies.VecTiles.transform.numeric_min_filter
     params:
       source_layer: buildings

spreadsheets/boundaries.csv

@@ -0,0 +1,21 @@
+kind,sort_key
+municipality,252
+county,254
+Admin-1 boundary,256
+state,256
+1st Order Admin Line,256
+Admin-1 region boundary,257
+Indefinite (please verify),259
+Indeterminant frontier,259
+Lease limit,259
+Overlay limit,259
+Line of control (please verify),260
+Disputed (please verify),261
+Admin-0 country,262
+country,262
+International boundary (verify),262
+city_wall,264
+retaining_wall,265
+snow_fence,266
+fence,267
+*,250

spreadsheets/buildings.csv

@@ -0,0 +1,2 @@
+kind,sort_key
+*,475

spreadsheets/earth.csv

@@ -0,0 +1,2 @@
+land,sort_key
+*,10

spreadsheets/landuse.csv

@@ -0,0 +1,94 @@
+kind,sort_key
+aerodrome,49
+allotments,83
+amusement_ride,97
+animal,92
+apron,51
+aquarium,46
+artwork,93
+attraction,94
+aviary,72
+beach,87
+breakwater,221
+bridge,226
+carousel,95
+cemetery,66
+cinema,69
+college,32
+commercial,35
+common,56
+conservation,24
+cutline,223
+dike,224
+enclosure,74
+farm,28
+farmland,29
+farmyard,61
+footway,100
+forest,30
+fuel,68
+garden,88
+generator,84
+glacier,12
+golf_course,47
+grass,78
+groyne,222
+hanami,86
+hospital,45
+industrial,23
+land,220
+library,70
+maze,85
+meadow,79
+military,37
+national_park,19
+nature_reserve,25
+park or protected land,17
+park,26
+park,33
+parking,64
+pedestrian,89
+petting_zoo,73
+pier,225
+pitch,91
+place_of_worship,65
+plant,77
+playground,90
+prison,38
+protected_area,18
+quarry,63
+railway,62
+recreation_ground,48
+residential,22
+resort,43
+retail,36
+roller_coaster,75
+runway,52
+rural,15
+school,67
+scrub,80
+sports_centre,54
+stadium,53
+substation,60
+summer_toboggan,76
+taxiway,50
+theatre,71
+theme_park,39
+tower,99
+trail_riding_station,44
+university,31
+urban area,13
+urban,14
+village_green,55
+wastewater_plant,57
+water_slide,96
+water_works,58
+wetland,81
+wilderness_hut,98
+wildlife_park,40
+winery,41
+winter_sports,27
+wood,82
+works,59
+zoo,42
+*,11

spreadsheets/roads.csv

@@ -0,0 +1,69 @@
+layer::int,kind,highway,type,railway,aeroway,service,aerialway,is_bridge,is_tunnel,sort_key
+7,*,*,*,*,*,*,*,*,*,447
+6,*,*,*,*,*,*,*,*,*,447
+5,*,*,*,*,*,*,*,*,*,447
+4,*,*,*,*,*,*,*,*,*,446
+3,*,*,*,*,*,*,*,*,*,445
+2,*,*,*,*,*,*,*,*,*,444
+1,*,*,*,*,*,*,*,*,*,443
+-1,*,*,*,*,*,*,*,*,*,304
+-2,*,*,*,*,*,*,*,*,*,303
+-3,*,*,*,*,*,*,*,*,*,302
+-4,*,*,*,*,*,*,*,*,*,301
+-5,*,*,*,*,*,*,*,*,*,300
+-6,*,*,*,*,*,*,*,*,*,300
+-7,*,*,*,*,*,*,*,*,*,300
+-,highway,motorway,*,*,*,*,*,-,-,381
+-,*,*,Major Highway;Beltway;Bypass,*,*,*,*,-,-,381
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,-,*,-,-,380
+-,major_road,trunk,*,*,*,*,*,-,-,379
+-,*,*,Secondary Highway,*,*,*,*,-,-,379
+-,major_road,primary,*,*,*,*,*,-,-,378
+-,*,*,Road,*,*,*,*,-,-,378
+-,major_road,secondary,*,*,*,*,*,-,-,377
+-,minor_road,*,*,*,runway,*,*,-,-,377
+-,major_road,tertiary,*,*,*,*,*,-,-,376
+-,minor_road,*,*,*,taxiway,*,*,-,-,376
+-,*,*,Track,*,*,*,*,-,-,376
+-,highway,motorway_link;trunk_link;primary_link;secondary_link;tertiary_link,*,*,*,*,*,-,-,375
+-,minor_road,residential;unclassified;road;living_street,*,*,*,*,*,-,-,360
+-,*,*,Unknown,*,*,*,*,-,-,360
+-,minor_road,service;minor,*,*,*,-,*,-,-,358
+-,minor_road,service;minor,*,*,*,*,*,-,-,356
+-,aerialway,*,*,*,*,*,gondola;cable_car,*,*,442
+-,aerialway,*,*,*,*,*,chair_lift,*,*,441
+-,aerialway,*,*,*,*,*,+,*,*,440
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,spur;siding,*,-,-,361
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,yard,*,-,-,359
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,+,*,-,-,357
+-,highway,motorway,*,*,*,*,*,yes,-,431
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,-,*,yes,-,430
+-,major_road,trunk,*,*,*,*,*,yes,-,429
+-,major_road,primary,*,*,*,*,*,yes,-,428
+-,major_road,secondary,*,*,*,*,*,yes,-,427
+-,minor_road,*,*,*,runway,*,*,yes,-,427
+-,major_road,tertiary,*,*,*,*,*,yes,-,426
+-,minor_road,*,*,*,taxiway,*,*,yes,-,426
+-,highway,motorway_link;trunk_link;primary_link;secondary_link;tertiary_link,*,*,*,*,*,yes,-,425
+-,minor_road,residential;unclassified;road;living_street,*,*,*,*,*,yes,-,410
+-,minor_road,service;minor,*,*,*,-,*,yes,-,408
+-,minor_road,service;minor,*,*,*,*,*,yes,-,406
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,spur;siding,*,yes,-,411
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,yard,*,yes,-,409
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,+,*,yes,-,407
+-,highway,motorway,*,*,*,*,*,-,yes,331
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,-,*,-,yes,330
+-,major_road,trunk,*,*,*,*,*,-,yes,329
+-,major_road,primary,*,*,*,*,*,-,yes,328
+-,major_road,secondary,*,*,*,*,*,-,yes,327
+-,minor_road,*,*,*,runway,*,*,-,yes,327
+-,major_road,tertiary,*,*,*,*,*,-,yes,326
+-,minor_road,*,*,*,taxiway,*,*,-,yes,326
+-,highway,motorway_link;trunk_link;primary_link;secondary_link;tertiary_link,*,*,*,*,*,-,yes,325
+-,minor_road,residential;unclassified;road;living_street,*,*,*,*,*,-,yes,310
+-,minor_road,service;minor,*,*,*,-,*,-,yes,308
+-,minor_road,service;minor,*,*,*,*,*,-,yes,306
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,spur;siding,*,-,yes,311
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,yard,*,-,yes,309
+-,rail,*,*,rail;tram;light_rail;narrow_gauge;monorail,*,+,*,-,yes,307
+-,*,*,*,*,*,*,*,*,*,355

spreadsheets/transit.csv

@@ -0,0 +1,5 @@
+kind,sort_key
+tram,415
+light_rail,415
+platform,385
+*,275

spreadsheets/water.csv

@@ -0,0 +1,16 @@
+kind,boundary,sort_key
+river,,201
+canal,,201
+stream,,201
+dam,,201
+ditch,,201
+drain,,201
+playa,,203
+lake,,204
+water,,204
+riverbank,,204
+reservoir,,204
+swimming_pool,,206
+ocean,yes,205
+*,yes,367
+*,*,200

test.py

@@ -84,6 +84,47 @@ def match_properties(actual, expected):
     return True
 
 
+# quantify how different a feature is from the expected feature properties.
+# this metric can be used to figure out what is a "near miss" in terms of
+# the match and output a more helpful error message.
+#
+# returns a pair of:
+#   - a numeric metric of the distance, where 0 is a match and larger
+#     numbers are further from a match.
+#   - an explanation of the "miss".
+def match_distance(actual, expected):
+    distance = 0
+    misses = dict()
+
+    for exp_k, exp_v in expected.iteritems():
+        v = actual.get(exp_k, None)
+        # normalise unicode values
+        if isinstance(v, unicode):
+            v = v.encode('utf-8')
+
+        if exp_v is not None:
+            if isinstance(exp_v, set):
+                if v not in exp_v:
+                    misses[exp_k] = "%r not in %r" % (v, exp_v)
+                    distance += 1
+
+            elif isinstance(exp_v, type):
+                if not isinstance(v, exp_v):
+                    misses[exp_k] = "%r not an instance of %r" % (v, exp_v)
+                    distance += 1
+
+            elif v != exp_v:
+                misses[exp_k] = "%r != %r" % (v, exp_v)
+                distance += 1
+
+        else:
+            if v is None:
+                misses[exp_k] = "missing"
+                distance += 1
+
+    return (distance, misses)
+
+
 @contextmanager
 def features_in_tile_layer(z, x, y, layer):
     url = config_url % {'layer': layer, 'z': z, 'x': x, 'y': y}
@@ -131,6 +172,26 @@ def count_matching(features, properties):
     return (num_features, num_matching)
 
 
+def closest_matching(features, properties):
+    """
+    Returns a pair containing the feature which most closely matches the
+    properties and a dict explaining the ways in which it didn't match.
+    """
+
+    min_distance = None
+    min_feature = None
+    min_misses = None
+
+    for f in features:
+        distance, misses = match_distance(f['properties'], properties)
+        if min_distance is None or distance < min_distance:
+            min_distance = distance
+            min_feature = f
+            min_misses = misses
+
+    return (min_feature, min_misses)
+
+
 def assert_has_feature(z, x, y, layer, properties):
     with features_in_tile_layer(z, x, y, layer) as features:
         num_features, num_matching = count_matching(
@@ -141,8 +202,10 @@ def assert_has_feature(z, x, y, layer, properties):
                 "%r (because layer %r was empty)" % (properties, layer)
 
         if num_matching == 0:
-            raise Exception, "Did not find feature including properties %r" \
-                % properties
+            closest, misses = closest_matching(features, properties)
+            raise Exception, "Did not find feature including properties " \
+                "%r. The closest match was %r: missed %r." \
+                % (properties, closest['properties'], misses)
 
 
 def assert_at_least_n_features(z, x, y, layer, properties, n):