diff --git a/docs/layers.md b/docs/layers.md index 045605cac..1dc660a9e 100644 --- a/docs/layers.md +++ b/docs/layers.md @@ -262,6 +262,8 @@ _TIP: Some `landuse` features only exist as point features in OpenStreetMap. Fin * `protect_class`: Common values include `1`, `2`, `3`, `4`, `5`, `6`. See [OSM wiki](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dprotected_area#Protect_classes_for_various_countries) for more information. * `operator`: e.g. `United States National Park Service`, `United States Forest Service`, `National Parks & Wildlife Service NSW`. +* `scalerank`: A positive integer zoom level, usually in the range 0 to 16. If this property is set on a feature at a particular zoom then it indicates that this feature is an appropriate size to display at this zoom. Features without this property set may be too small, or missing "importance raising" properties. +* `tier`: An integer between 1 and 6, with 1 being "more important" and 6 being "less important". #### Landuse kind values: diff --git a/integration-test/473-landuse-scalerank.py b/integration-test/473-landuse-scalerank.py new file mode 100644 index 000000000..147bce97d --- /dev/null +++ b/integration-test/473-landuse-scalerank.py @@ -0,0 +1,26 @@ +# http://www.openstreetmap.org/way/167274589 +# area 300363008 +assert_has_feature( + 16, 10818, 21900, 'landuse', + { 'kind': 'national_park', 'id': 167274589, 'tier': 1, + 'scalerank': 3 }) + + +# http://www.openstreetmap.org/relation/921675 +# area 30089300 +assert_has_feature( + 16, 14579, 29651, 'landuse', + { 'kind': 'national_park', 'id': -921675, 'tier': 1, + 'scalerank': 8 }) +assert_has_feature( + 7, 28, 57, 'landuse', + { 'kind': 'national_park', 'id': -921675, + 'tier': type(None), 'scalerank': type(None) }) + +# this is USFS, so demoted to scalerank 2 :-( +# http://www.openstreetmap.org/way/34416231 +# area 86685400 +assert_has_feature( + 16, 18270, 25157, 'landuse', + { 'kind': 'national_park', 'id': 34416231, + 'tier': 2, 'scalerank': 8 }) diff --git a/queries.yaml b/queries.yaml index 683e79050..c3a35a091 100644 --- a/queries.yaml +++ b/queries.yaml @@ -205,7 +205,6 @@ post_process: path: spreadsheets/sort_key/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -214,7 +213,6 @@ post_process: path: spreadsheets/sort_key/roads.csv params: source_layer: roads - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -223,7 +221,6 @@ post_process: path: spreadsheets/sort_key/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -232,7 +229,6 @@ post_process: path: spreadsheets/sort_key/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -241,7 +237,6 @@ post_process: path: spreadsheets/sort_key/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -250,7 +245,6 @@ post_process: path: spreadsheets/sort_key/transit.csv params: source_layer: transit - target_value_type: int # this needs to run before the water sort_key csv matcher - fn: vectordatasource.transform.exterior_boundaries params: @@ -273,7 +267,6 @@ post_process: path: spreadsheets/sort_key/water.csv params: source_layer: water - target_value_type: int # assign `scalerank` to features - fn: vectordatasource.transform.csv_match_properties @@ -284,7 +277,6 @@ post_process: path: spreadsheets/scalerank/water.csv params: source_layer: water - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -293,7 +285,6 @@ post_process: path: spreadsheets/scalerank/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -302,7 +293,6 @@ post_process: path: spreadsheets/scalerank/places.csv params: source_layer: places - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -311,7 +301,6 @@ post_process: path: spreadsheets/scalerank/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -320,7 +309,6 @@ post_process: path: spreadsheets/scalerank/roads.csv params: source_layer: roads - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -329,7 +317,6 @@ post_process: path: spreadsheets/scalerank/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -338,7 +325,6 @@ post_process: path: spreadsheets/scalerank/pois.csv params: source_layer: pois - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -347,7 +333,6 @@ post_process: path: spreadsheets/scalerank/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -356,7 +341,6 @@ post_process: path: spreadsheets/scalerank/transit.csv params: source_layer: transit - target_value_type: int # add minimum pixel thresholds per layer # this should run after all the `scalerank`s have been assigned to features @@ -370,7 +354,6 @@ post_process: path: spreadsheets/mz_min_pixels/water.csv params: source_layer: water - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -379,7 +362,6 @@ post_process: path: spreadsheets/mz_min_pixels/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -388,7 +370,6 @@ post_process: path: spreadsheets/mz_min_pixels/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -397,7 +378,6 @@ post_process: path: spreadsheets/mz_min_pixels/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -406,7 +386,6 @@ post_process: path: spreadsheets/mz_min_pixels/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -415,7 +394,6 @@ post_process: path: spreadsheets/mz_min_pixels/transit.csv params: source_layer: transit - target_value_type: int # drop all polygonal features that don't meet the minimum area # threshold. this depends on the mz_min_pixels property to be set on diff --git a/spreadsheets/mz_min_pixels/boundaries.csv b/spreadsheets/mz_min_pixels/boundaries.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/boundaries.csv +++ b/spreadsheets/mz_min_pixels/boundaries.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/buildings.csv b/spreadsheets/mz_min_pixels/buildings.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/buildings.csv +++ b/spreadsheets/mz_min_pixels/buildings.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/earth.csv b/spreadsheets/mz_min_pixels/earth.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/earth.csv +++ b/spreadsheets/mz_min_pixels/earth.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/landuse.csv b/spreadsheets/mz_min_pixels/landuse.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/landuse.csv +++ b/spreadsheets/mz_min_pixels/landuse.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/transit.csv b/spreadsheets/mz_min_pixels/transit.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/transit.csv +++ b/spreadsheets/mz_min_pixels/transit.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/water.csv b/spreadsheets/mz_min_pixels/water.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/water.csv +++ b/spreadsheets/mz_min_pixels/water.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/scalerank/boundaries.csv b/spreadsheets/scalerank/boundaries.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/boundaries.csv +++ b/spreadsheets/scalerank/boundaries.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/buildings.csv b/spreadsheets/scalerank/buildings.csv index a96d80d4a..ab706dca4 100644 --- a/spreadsheets/scalerank/buildings.csv +++ b/spreadsheets/scalerank/buildings.csv @@ -1,4 +1,4 @@ -area::int,height::float,volume::int,landuse_kind,scalerank +area::int,height::float,volume::int,landuse_kind,output:scalerank::int >=100000,*,*,*,1 *,>=250,*,*,1 *,*,>=300000,*,1 diff --git a/spreadsheets/scalerank/earth.csv b/spreadsheets/scalerank/earth.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/earth.csv +++ b/spreadsheets/scalerank/earth.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/landuse.csv b/spreadsheets/scalerank/landuse.csv index d755d3f89..a1d041498 100644 --- a/spreadsheets/scalerank/landuse.csv +++ b/spreadsheets/scalerank/landuse.csv @@ -1 +1,76 @@ -kind,scalerank +kind,operator,zoom::int,area::int,output:tier::int,output:scalerank::int +national_park;battlefield,!United States Forest Service,>=3,>=300000000,1,3 +national_park;battlefield,!United States Forest Service,>=4,>=300000000,1,4 +national_park;battlefield,!United States Forest Service,>=5,>=150000000,1,5 +national_park;battlefield,!United States Forest Service,>=6,>=150000000,1,6 +national_park;battlefield,!United States Forest Service,>=7,>=100000000,1,7 +national_park;battlefield,!United States Forest Service,>=8,>=10000000,1,8 +national_park;battlefield,!United States Forest Service,>=9,>=5000000,1,9 +national_park;battlefield,!United States Forest Service,>=10,>=1000000,1,10 +national_park;battlefield,!United States Forest Service,>=11,>=500000,1,11 +national_park;battlefield,!United States Forest Service,>=12,>=500000,1,12 +national_park;battlefield,!United States Forest Service,>=13,>=100000,1,13 +national_park;battlefield,!United States Forest Service,>=14,>=50000,1,14 +national_park;battlefield,!United States Forest Service,>=15,>=2000,1,15 +national_park;battlefield,!United States Forest Service,>=16,+,1,16 +park;national_park,!United States National Park Service,>=4,>=1000000000,2,4 +park;national_park,!United States National Park Service,>=5,>=1000000000,2,5 +park;national_park,!United States National Park Service,>=6,>=150000000,2,6 +park;national_park,!United States National Park Service,>=7,>=100000000,2,7 +park;national_park,!United States National Park Service,>=8,>=10000000,2,8 +park;national_park,!United States National Park Service,>=9,>=5000000,2,9 +park;national_park,!United States National Park Service,>=10,>=1000000,2,10 +park;national_park,!United States National Park Service,>=11,>=500000,2,11 +park;national_park,!United States National Park Service,>=12,>=250000,2,12 +park;national_park,!United States National Park Service,>=13,>=100000,2,13 +park;national_park,!United States National Park Service,>=14,>=50000,2,14 +park;national_park,!United States National Park Service,>=15,>=2000,2,15 +park;national_park,!United States National Park Service,>=16,+,2,16 +forest;wood;natural_forest;natural_wood,United States Forest Service,>=6,>=150000000,2,6 +forest;wood;natural_forest;natural_wood,United States Forest Service,>=7,>=100000000,2,7 +forest;wood;natural_forest;natural_wood,United States Forest Service,>=8,>=10000000,2,8 +forest;wood;natural_forest;natural_wood,United States Forest Service,>=9,>=5000000,2,9 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=4,>=1000000000,2,4 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=5,>=1000000000,2,5 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=6,>=150000000,2,6 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=7,>=100000000,2,7 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=8,>=10000000,2,8 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=9,>=5000000,2,9 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=10,>=1000000,2,10 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=11,>=500000,2,11 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=12,>=250000,2,12 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=13,>=100000,2,13 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=14,>=50000,2,14 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=15,>=2000,2,15 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;natural_forest;natural_wood;farm;farmland,,>=16,+,2,16 +aerodrome;military;university;college,,>=8,>=10000000,3,8 +aerodrome;military;university;college,,>=9,>=5000000,3,9 +aerodrome;military;university;college,,>=10,>=1000000,3,10 +aerodrome;military;university;college,,>=11,>=500000,3,11 +aerodrome;military;university;college,,>=12,>=500000,3,12 +aerodrome;military;university;college,,>=13,>=100000,3,13 +aerodrome;military;university;college,,>=14,>=50000,3,14 +aerodrome;military;university;college,,>=15,>=2000,3,15 +aerodrome;military;university;college,,>=16,+,3,16 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=10,>=1000000,4,10 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=11,>=500000,4,11 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=12,>=500000,4,12 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=13,>=100000,4,13 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=14,>=50000,4,14 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=15,>=2000,4,15 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=16,+,4,16 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=10,>=1000000,5,10 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=11,>=400000,5,11 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=12,>=200000,5,12 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=13,>=50000,5,13 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=14,>=20000,5,14 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=15,>=2000,5,15 +theme_park;resort;aquarium;winery;maze;beach;glacier,,>=16,+,5,16 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=12,>=500000,6,12 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=13,>=100000,6,13 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=14,>=50000,6,14 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=15,>=2000,6,15 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=16,+,6,16 +parking,,>=14,>=50000,6,14 +parking,,>=15,>=5000,6,15 +parking,,>=16,+,6,16 diff --git a/spreadsheets/scalerank/places.csv b/spreadsheets/scalerank/places.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/places.csv +++ b/spreadsheets/scalerank/places.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/pois.csv b/spreadsheets/scalerank/pois.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/pois.csv +++ b/spreadsheets/scalerank/pois.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/roads.csv b/spreadsheets/scalerank/roads.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/roads.csv +++ b/spreadsheets/scalerank/roads.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/transit.csv b/spreadsheets/scalerank/transit.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/transit.csv +++ b/spreadsheets/scalerank/transit.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/water.csv b/spreadsheets/scalerank/water.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/water.csv +++ b/spreadsheets/scalerank/water.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/sort_key/boundaries.csv b/spreadsheets/sort_key/boundaries.csv index 32961158d..474e6c961 100644 --- a/spreadsheets/sort_key/boundaries.csv +++ b/spreadsheets/sort_key/boundaries.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int municipality,252 county,254 Admin-1 boundary,256 diff --git a/spreadsheets/sort_key/buildings.csv b/spreadsheets/sort_key/buildings.csv index 83f956d71..ae67dd88e 100644 --- a/spreadsheets/sort_key/buildings.csv +++ b/spreadsheets/sort_key/buildings.csv @@ -1,2 +1,2 @@ -kind,sort_key +kind,output:sort_key::int *,475 diff --git a/spreadsheets/sort_key/earth.csv b/spreadsheets/sort_key/earth.csv index ec4ba55b5..3a0dd92d3 100644 --- a/spreadsheets/sort_key/earth.csv +++ b/spreadsheets/sort_key/earth.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int cliff,227 arete,228 earth,10 diff --git a/spreadsheets/sort_key/landuse.csv b/spreadsheets/sort_key/landuse.csv index b8b204544..69e1f90a6 100644 --- a/spreadsheets/sort_key/landuse.csv +++ b/spreadsheets/sort_key/landuse.csv @@ -1,4 +1,4 @@ -kind,geometrytype,sort_key +kind,geometrytype,output:sort_key::int fence,*,269 snow_fence,*,268 retaining_wall,*,267 diff --git a/spreadsheets/sort_key/roads.csv b/spreadsheets/sort_key/roads.csv index fe7006959..171af5bb9 100644 --- a/spreadsheets/sort_key/roads.csv +++ b/spreadsheets/sort_key/roads.csv @@ -1,4 +1,4 @@ -zoom::int,layer::int,kind,kind_detail,service,is_bridge,is_tunnel,sort_key +zoom::int,layer::int,kind,kind_detail,service,is_bridge,is_tunnel,output:sort_key::int >=15,7,*,*,*,*,*,447 >=15,6,*,*,*,*,*,447 >=15,5,*,*,*,*,*,447 diff --git a/spreadsheets/sort_key/transit.csv b/spreadsheets/sort_key/transit.csv index f669098f5..cde6ac07e 100644 --- a/spreadsheets/sort_key/transit.csv +++ b/spreadsheets/sort_key/transit.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int light_rail,418 tram,417 funicular,416 diff --git a/spreadsheets/sort_key/water.csv b/spreadsheets/sort_key/water.csv index 8cd309b4e..20255bfa9 100644 --- a/spreadsheets/sort_key/water.csv +++ b/spreadsheets/sort_key/water.csv @@ -1,4 +1,4 @@ -kind,boundary,sort_key +kind,boundary,output:sort_key::int river,,201 canal,,201 stream,,201 diff --git a/test/test_transform.py b/test/test_transform.py index c1880bb0c..96414e7a4 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -21,8 +21,8 @@ def _call_fut(self, props): match_result = self.matcher(shape, props, z) if match_result is None: return None - k, v = match_result - return float(v) + kvs = match_result + return float(kvs.get('scalerank')) def test_area_most_important(self): props = dict(area=1000000) @@ -275,14 +275,15 @@ def test_geometry_type(self): zoom = 16 sort_key_result = self.matcher(shape, props, zoom) self.assertIsNotNone(sort_key_result) - _, sort_key = sort_key_result - self.assertEquals(int(sort_key), 265) + sort_key = sort_key_result.get('sort_key') + self.assertEquals(type(sort_key), int) + self.assertEquals(sort_key, 265) shape = shapely.geometry.Polygon([(0, 0), (1, 1), (0, 1), (0, 0)]) sort_key_result = self.matcher(shape, props, zoom) self.assertIsNotNone(sort_key_result) - _, sort_key = sort_key_result - self.assertEquals(int(sort_key), 223) + sort_key = sort_key_result.get('sort_key') + self.assertEquals(sort_key, 223) class BuildingsUnifyTest(unittest.TestCase): diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index f520130c8..d61e78eef 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -3289,7 +3289,7 @@ def match(self, other): return other in self.values def __repr__(self): - return repr(self.value) + return repr(self.values) class _GreaterThanEqualMatcher(object): @@ -3351,12 +3351,47 @@ def _parse_kt(key_type): return (kt[0], fn) +def _parse_csv_header(header): + """ + Parse a CSV column header into a tuple containing: + * col_is_target: True if the column is an output / target column. This is + triggered if the header begins with "output:". + * key: The string name of the column. + * typ: A function returning the a value of the type of the column given a + string. This is used to ensure that integer columns are kept as + integers, rather than the strings they're read from the file as. + """ + + # target (output) columns start with 'output:' + col_is_target = header.startswith('output:') + if col_is_target: + header = header[len('output:'):] + + # all columns can have an optional type after ::, + # e.g: key::int if it's supposed to be an integer. + key, typ = _parse_kt(header) + + return (col_is_target, key, typ) + + class CSVMatcher(object): def __init__(self, fh): - keys = None - types = [] + # collects together column information in the form of a tuple + # (is_target, key, typ) where is_target is a boolean which is true + # if the column is a target (i.e: output) column, or false if it is + # a column to be matched. key is a string key, and typ is a function + # which returns a type of that column given a string as input. + # typically this is int(), float(), etc... + cols = None + + # each row is a potential match, and rows keeps a tuple for each of + # (matchers, target values), where matchers is an array of matcher + # objects, one for each non-target column. rows = [] + # keep a set of static matchers to cut down the number of objects. + # these matchers are "constant" and don't take any input, so this is + # a safe operation. self.static_any = _AnyMatcher() self.static_none = _NoneMatcher() self.static_some = _SomeMatcher() @@ -3365,23 +3400,35 @@ def __init__(self, fh): # CSV - allow whitespace after the comma reader = csv.reader(fh, skipinitialspace=True) for row in reader: - if keys is None: - target_key = row.pop(-1) - keys = [] - for key_type in row: - key, typ = _parse_kt(key_type) - keys.append(key) - types.append(typ) + if cols is None: + cols = [_parse_csv_header(hdr) for hdr in row] else: - target_val = row.pop(-1) - for i in range(0, len(row)): - row[i] = self._match_val(row[i], types[i]) - rows.append((row, target_val)) + rows.append(self._parse_csv_row(row, cols)) - self.keys = keys + # keys is the string key for each matcher, so that values can be + # extracted from the object being matched in the same order as the + # array of matcher objects. + self.keys = [key for is_target, key, typ in cols] self.rows = rows - self.target_key = target_key + + def _parse_csv_row(self, row, cols): + # match_row is the array of matcher objects which will be + # called with each column's value. + match_row = [] + + # target_vals is the dict of values to return to the caller + # for the first row where all the matchers return true. + target_vals = {} + + for (is_target, key, typ), val in zip(cols, row): + if is_target: + target_vals[key] = typ(val) + + else: + match_row.append(self._match_val(val, typ)) + + return (match_row, target_vals) def _match_val(self, v, typ): if v == '*': @@ -3422,9 +3469,9 @@ def __call__(self, shape, properties, zoom): else: val = properties.get(key) vals.append(val) - for row, target_val in self.rows: + for row, target_vals in self.rows: if all([a.match(b) for (a, b) in zip(row, vals)]): - return (self.target_key, target_val) + return target_vals return None @@ -3440,7 +3487,6 @@ def csv_match_properties(ctx): source_layer = ctx.params.get('source_layer') start_zoom = ctx.params.get('start_zoom', 0) end_zoom = ctx.params.get('end_zoom') - target_value_type = ctx.params.get('target_value_type') matcher = ctx.resources.get('matcher') assert source_layer, 'csv_match_properties: missing source layer' @@ -3456,16 +3502,10 @@ def csv_match_properties(ctx): if layer is None: return None - def _type_cast(v): - if target_value_type == 'int': - return int(v) - return v - for shape, props, fid in layer['features']: m = matcher(shape, props, zoom) if m is not None: - k, v = m - props[k] = _type_cast(v) + props.update(m) return layer