From 57aeb9cd139c699840940ea8aa96f4b92db92788 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 23 Aug 2016 14:08:41 +0100 Subject: [PATCH] Allow landuse scalerank CSV to also output tier. CSVMatcher now uses the header key to determine if the column is an output column - those with the prefix "output:" are used. This allows the use of multiple outputs in the same CSV, and that is used to output both the scalerank and tier from the landuse CSV. --- integration-test/473-landuse-scalerank.py | 12 +- queries.yaml | 22 ---- spreadsheets/mz_min_pixels/boundaries.csv | 2 +- spreadsheets/mz_min_pixels/buildings.csv | 2 +- spreadsheets/mz_min_pixels/earth.csv | 2 +- spreadsheets/mz_min_pixels/landuse.csv | 2 +- spreadsheets/mz_min_pixels/transit.csv | 2 +- spreadsheets/mz_min_pixels/water.csv | 2 +- spreadsheets/scalerank/boundaries.csv | 2 +- spreadsheets/scalerank/buildings.csv | 2 +- spreadsheets/scalerank/earth.csv | 2 +- spreadsheets/scalerank/landuse.csv | 144 +++++++++++----------- spreadsheets/scalerank/places.csv | 2 +- spreadsheets/scalerank/pois.csv | 2 +- spreadsheets/scalerank/roads.csv | 2 +- spreadsheets/scalerank/transit.csv | 2 +- spreadsheets/scalerank/water.csv | 2 +- spreadsheets/sort_key/boundaries.csv | 2 +- spreadsheets/sort_key/buildings.csv | 2 +- spreadsheets/sort_key/earth.csv | 2 +- spreadsheets/sort_key/landuse.csv | 2 +- spreadsheets/sort_key/roads.csv | 2 +- spreadsheets/sort_key/transit.csv | 2 +- spreadsheets/sort_key/water.csv | 2 +- test/test_transform.py | 13 +- vectordatasource/transform.py | 73 +++++++---- 26 files changed, 158 insertions(+), 148 deletions(-) diff --git a/integration-test/473-landuse-scalerank.py b/integration-test/473-landuse-scalerank.py index b22b1a867..147bce97d 100644 --- a/integration-test/473-landuse-scalerank.py +++ b/integration-test/473-landuse-scalerank.py @@ -2,21 +2,25 @@ # area 300363008 assert_has_feature( 16, 10818, 21900, 'landuse', - { 'kind': 'national_park', 'id': 167274589, 'scalerank': 1 }) + { 'kind': 'national_park', 'id': 167274589, 'tier': 1, + 'scalerank': 3 }) # http://www.openstreetmap.org/relation/921675 # area 30089300 assert_has_feature( 16, 14579, 29651, 'landuse', - { 'kind': 'national_park', 'id': -921675, 'scalerank': 1 }) + { 'kind': 'national_park', 'id': -921675, 'tier': 1, + 'scalerank': 8 }) assert_has_feature( 7, 28, 57, 'landuse', - { 'kind': 'national_park', 'id': -921675, 'scalerank': type(None) }) + { 'kind': 'national_park', 'id': -921675, + 'tier': type(None), 'scalerank': type(None) }) # this is USFS, so demoted to scalerank 2 :-( # http://www.openstreetmap.org/way/34416231 # area 86685400 assert_has_feature( 16, 18270, 25157, 'landuse', - { 'kind': 'national_park', 'id': 34416231, 'scalerank': 2 }) + { 'kind': 'national_park', 'id': 34416231, + 'tier': 2, 'scalerank': 8 }) diff --git a/queries.yaml b/queries.yaml index 683e79050..c3a35a091 100644 --- a/queries.yaml +++ b/queries.yaml @@ -205,7 +205,6 @@ post_process: path: spreadsheets/sort_key/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -214,7 +213,6 @@ post_process: path: spreadsheets/sort_key/roads.csv params: source_layer: roads - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -223,7 +221,6 @@ post_process: path: spreadsheets/sort_key/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -232,7 +229,6 @@ post_process: path: spreadsheets/sort_key/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -241,7 +237,6 @@ post_process: path: spreadsheets/sort_key/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -250,7 +245,6 @@ post_process: path: spreadsheets/sort_key/transit.csv params: source_layer: transit - target_value_type: int # this needs to run before the water sort_key csv matcher - fn: vectordatasource.transform.exterior_boundaries params: @@ -273,7 +267,6 @@ post_process: path: spreadsheets/sort_key/water.csv params: source_layer: water - target_value_type: int # assign `scalerank` to features - fn: vectordatasource.transform.csv_match_properties @@ -284,7 +277,6 @@ post_process: path: spreadsheets/scalerank/water.csv params: source_layer: water - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -293,7 +285,6 @@ post_process: path: spreadsheets/scalerank/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -302,7 +293,6 @@ post_process: path: spreadsheets/scalerank/places.csv params: source_layer: places - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -311,7 +301,6 @@ post_process: path: spreadsheets/scalerank/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -320,7 +309,6 @@ post_process: path: spreadsheets/scalerank/roads.csv params: source_layer: roads - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -329,7 +317,6 @@ post_process: path: spreadsheets/scalerank/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -338,7 +325,6 @@ post_process: path: spreadsheets/scalerank/pois.csv params: source_layer: pois - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -347,7 +333,6 @@ post_process: path: spreadsheets/scalerank/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -356,7 +341,6 @@ post_process: path: spreadsheets/scalerank/transit.csv params: source_layer: transit - target_value_type: int # add minimum pixel thresholds per layer # this should run after all the `scalerank`s have been assigned to features @@ -370,7 +354,6 @@ post_process: path: spreadsheets/mz_min_pixels/water.csv params: source_layer: water - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -379,7 +362,6 @@ post_process: path: spreadsheets/mz_min_pixels/earth.csv params: source_layer: earth - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -388,7 +370,6 @@ post_process: path: spreadsheets/mz_min_pixels/landuse.csv params: source_layer: landuse - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -397,7 +378,6 @@ post_process: path: spreadsheets/mz_min_pixels/buildings.csv params: source_layer: buildings - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -406,7 +386,6 @@ post_process: path: spreadsheets/mz_min_pixels/boundaries.csv params: source_layer: boundaries - target_value_type: int - fn: vectordatasource.transform.csv_match_properties resources: matcher: @@ -415,7 +394,6 @@ post_process: path: spreadsheets/mz_min_pixels/transit.csv params: source_layer: transit - target_value_type: int # drop all polygonal features that don't meet the minimum area # threshold. this depends on the mz_min_pixels property to be set on diff --git a/spreadsheets/mz_min_pixels/boundaries.csv b/spreadsheets/mz_min_pixels/boundaries.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/boundaries.csv +++ b/spreadsheets/mz_min_pixels/boundaries.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/buildings.csv b/spreadsheets/mz_min_pixels/buildings.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/buildings.csv +++ b/spreadsheets/mz_min_pixels/buildings.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/earth.csv b/spreadsheets/mz_min_pixels/earth.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/earth.csv +++ b/spreadsheets/mz_min_pixels/earth.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/landuse.csv b/spreadsheets/mz_min_pixels/landuse.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/landuse.csv +++ b/spreadsheets/mz_min_pixels/landuse.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/transit.csv b/spreadsheets/mz_min_pixels/transit.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/transit.csv +++ b/spreadsheets/mz_min_pixels/transit.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/mz_min_pixels/water.csv b/spreadsheets/mz_min_pixels/water.csv index 550895bc1..365d07446 100644 --- a/spreadsheets/mz_min_pixels/water.csv +++ b/spreadsheets/mz_min_pixels/water.csv @@ -1 +1 @@ -scalerank::int,mz_min_pixels +scalerank::int,output:mz_min_pixels::int diff --git a/spreadsheets/scalerank/boundaries.csv b/spreadsheets/scalerank/boundaries.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/boundaries.csv +++ b/spreadsheets/scalerank/boundaries.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/buildings.csv b/spreadsheets/scalerank/buildings.csv index a96d80d4a..ab706dca4 100644 --- a/spreadsheets/scalerank/buildings.csv +++ b/spreadsheets/scalerank/buildings.csv @@ -1,4 +1,4 @@ -area::int,height::float,volume::int,landuse_kind,scalerank +area::int,height::float,volume::int,landuse_kind,output:scalerank::int >=100000,*,*,*,1 *,>=250,*,*,1 *,*,>=300000,*,1 diff --git a/spreadsheets/scalerank/earth.csv b/spreadsheets/scalerank/earth.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/earth.csv +++ b/spreadsheets/scalerank/earth.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/landuse.csv b/spreadsheets/scalerank/landuse.csv index fdec9bcbb..52248ceee 100644 --- a/spreadsheets/scalerank/landuse.csv +++ b/spreadsheets/scalerank/landuse.csv @@ -1,72 +1,72 @@ -kind,operator,zoom::int,area::int,scalerank -national_park;park or protected land;battlefield,!United States Forest Service,3,>=300000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,4,>=300000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,5,>=150000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,6,>=150000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,7,>=100000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,8,>=10000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,9,>=5000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,10,>=1000000,1 -national_park;park or protected land;battlefield,!United States Forest Service,11,>=500000,1 -national_park;park or protected land;battlefield,!United States Forest Service,12,>=500000,1 -national_park;park or protected land;battlefield,!United States Forest Service,13,>=100000,1 -national_park;park or protected land;battlefield,!United States Forest Service,14,>=50000,1 -national_park;park or protected land;battlefield,!United States Forest Service,15,>=2000,1 -national_park;park or protected land;battlefield,!United States Forest Service,>=16,+,1 -national_park;park or protected land,!United States National Park Service,4,>=1000000000,2 -national_park;park or protected land,!United States National Park Service,5,>=1000000000,2 -national_park;park or protected land,!United States National Park Service,6,>=150000000,2 -national_park;park or protected land,!United States National Park Service,7,>=100000000,2 -national_park;park or protected land,!United States National Park Service,8,>=10000000,2 -national_park;park or protected land,!United States National Park Service,9,>=5000000,2 -national_park;park or protected land,!United States National Park Service,10,>=1000000,2 -national_park;park or protected land,!United States National Park Service,11,>=500000,2 -national_park;park or protected land,!United States National Park Service,12,>=250000,2 -national_park;park or protected land,!United States National Park Service,13,>=100000,2 -national_park;park or protected land,!United States National Park Service,14,>=50000,2 -national_park;park or protected land,!United States National Park Service,15,>=2000,2 -national_park;park or protected land,!United States National Park Service,>=16,+,2 -conservation;protected_area;nature_reserve;urban;rural;residential,,4,>=1000000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential,,5,>=1000000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood,,6,>=150000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,7,>=100000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,8,>=10000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,9,>=5000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,10,>=1000000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,11,>=500000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,12,>=250000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,13,>=100000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,14,>=50000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,15,>=2000,2 -conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=16,+,2 -aerodrome;military;university;college,,8,>=10000000,3 -aerodrome;military;university;college,,9,>=5000000,3 -aerodrome;military;university;college,,10,>=1000000,3 -aerodrome;military;university;college,,11,>=500000,3 -aerodrome;military;university;college,,12,>=500000,3 -aerodrome;military;university;college,,13,>=100000,3 -aerodrome;military;university;college,,14,>=50000,3 -aerodrome;military;university;college,,15,>=2000,3 -aerodrome;military;university;college,,>=16,+,3 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,10,>=1000000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,11,>=500000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,12,>=500000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,13,>=100000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,14,>=50000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,15,>=2000,4 -"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=16,+,4 -theme_park;resort;aquarium;winery;maze;beach,,10,>=1000000,5 -theme_park;resort;aquarium;winery;maze;beach,,11,>=400000,5 -theme_park;resort;aquarium;winery;maze;beach,,12,>=200000,5 -theme_park;resort;aquarium;winery;maze;beach,,13,>=50000,5 -theme_park;resort;aquarium;winery;maze;beach,,14,>=20000,5 -theme_park;resort;aquarium;winery;maze;beach,,15,>=2000,5 -theme_park;resort;aquarium;winery;maze;beach,,>=16,+,5 -garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,12,>=500000,6 -garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,13,>=100000,6 -garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,14,>=50000,6 -garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,15,>=2000,6 -garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=16,+,6 -parking,,14,>=50000,6 -parking,,15,>=5000,6 -parking,,>=16,+,6 +kind,operator,zoom::int,area::int,output:tier::int,output:scalerank::int +national_park;park or protected land;battlefield,!United States Forest Service,>=3,>=300000000,1,3 +national_park;park or protected land;battlefield,!United States Forest Service,>=4,>=300000000,1,4 +national_park;park or protected land;battlefield,!United States Forest Service,>=5,>=150000000,1,5 +national_park;park or protected land;battlefield,!United States Forest Service,>=6,>=150000000,1,6 +national_park;park or protected land;battlefield,!United States Forest Service,>=7,>=100000000,1,7 +national_park;park or protected land;battlefield,!United States Forest Service,>=8,>=10000000,1,8 +national_park;park or protected land;battlefield,!United States Forest Service,>=9,>=5000000,1,9 +national_park;park or protected land;battlefield,!United States Forest Service,>=10,>=1000000,1,10 +national_park;park or protected land;battlefield,!United States Forest Service,>=11,>=500000,1,11 +national_park;park or protected land;battlefield,!United States Forest Service,>=12,>=500000,1,12 +national_park;park or protected land;battlefield,!United States Forest Service,>=13,>=100000,1,13 +national_park;park or protected land;battlefield,!United States Forest Service,>=14,>=50000,1,14 +national_park;park or protected land;battlefield,!United States Forest Service,>=15,>=2000,1,15 +national_park;park or protected land;battlefield,!United States Forest Service,>=16,+,1,16 +national_park;park or protected land,!United States National Park Service,>=4,>=1000000000,2,4 +national_park;park or protected land,!United States National Park Service,>=5,>=1000000000,2,5 +national_park;park or protected land,!United States National Park Service,>=6,>=150000000,2,6 +national_park;park or protected land,!United States National Park Service,>=7,>=100000000,2,7 +national_park;park or protected land,!United States National Park Service,>=8,>=10000000,2,8 +national_park;park or protected land,!United States National Park Service,>=9,>=5000000,2,9 +national_park;park or protected land,!United States National Park Service,>=10,>=1000000,2,10 +national_park;park or protected land,!United States National Park Service,>=11,>=500000,2,11 +national_park;park or protected land,!United States National Park Service,>=12,>=250000,2,12 +national_park;park or protected land,!United States National Park Service,>=13,>=100000,2,13 +national_park;park or protected land,!United States National Park Service,>=14,>=50000,2,14 +national_park;park or protected land,!United States National Park Service,>=15,>=2000,2,15 +national_park;park or protected land,!United States National Park Service,>=16,+,2,16 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=4,>=1000000000,2,4 +conservation;protected_area;nature_reserve;urban;rural;residential,,>=5,>=1000000000,2,5 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood,,>=6,>=150000000,2,6 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,>=7,>=100000000,2,7 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,>=8,>=10000000,2,8 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reserve,,>=9,>=5000000,2,9 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=10,>=1000000,2,10 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=11,>=500000,2,11 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=12,>=250000,2,12 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=13,>=100000,2,13 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=14,>=50000,2,14 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=15,>=2000,2,15 +conservation;protected_area;nature_reserve;urban;rural;residential;forest;wood;nature_reservefarm;farmland,,>=16,+,2,16 +aerodrome;military;university;college,,>=8,>=10000000,3,8 +aerodrome;military;university;college,,>=9,>=5000000,3,9 +aerodrome;military;university;college,,>=10,>=1000000,3,10 +aerodrome;military;university;college,,>=11,>=500000,3,11 +aerodrome;military;university;college,,>=12,>=500000,3,12 +aerodrome;military;university;college,,>=13,>=100000,3,13 +aerodrome;military;university;college,,>=14,>=50000,3,14 +aerodrome;military;university;college,,>=15,>=2000,3,15 +aerodrome;military;university;college,,>=16,+,3,16 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=10,>=1000000,4,10 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=11,>=500000,4,11 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=12,>=500000,4,12 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=13,>=100000,4,13 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=14,>=50000,4,14 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=15,>=2000,4,15 +"cemetery;commercial;golf_course;hospital;industrial;plant;generator;substation;railway;sports_centre;recreation_ground;retail;stadium;zoo;wildlife_park;winter_sports;pier;wastewater_plant,works;bridge;tower;breakwater;water_works;groyne;dike;cutline",,>=16,+,4,16 +theme_park;resort;aquarium;winery;maze;beach,,>=10,>=1000000,5,10 +theme_park;resort;aquarium;winery;maze;beach,,>=11,>=400000,5,11 +theme_park;resort;aquarium;winery;maze;beach,,>=12,>=200000,5,12 +theme_park;resort;aquarium;winery;maze;beach,,>=13,>=50000,5,13 +theme_park;resort;aquarium;winery;maze;beach,,>=14,>=20000,5,14 +theme_park;resort;aquarium;winery;maze;beach,,>=15,>=2000,5,15 +theme_park;resort;aquarium;winery;maze;beach,,>=16,+,5,16 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=12,>=500000,6,12 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=13,>=100000,6,13 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=14,>=50000,6,14 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=15,>=2000,6,15 +garden;allotments;pedestrian;common;pitch;place_of_worship;playground;school;attraction;artwork;wilderness_hut;hanami;tree_row;hedge,,>=16,+,6,16 +parking,,>=14,>=50000,6,14 +parking,,>=15,>=5000,6,15 +parking,,>=16,+,6,16 diff --git a/spreadsheets/scalerank/places.csv b/spreadsheets/scalerank/places.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/places.csv +++ b/spreadsheets/scalerank/places.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/pois.csv b/spreadsheets/scalerank/pois.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/pois.csv +++ b/spreadsheets/scalerank/pois.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/roads.csv b/spreadsheets/scalerank/roads.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/roads.csv +++ b/spreadsheets/scalerank/roads.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/transit.csv b/spreadsheets/scalerank/transit.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/transit.csv +++ b/spreadsheets/scalerank/transit.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/scalerank/water.csv b/spreadsheets/scalerank/water.csv index d755d3f89..c0b37e276 100644 --- a/spreadsheets/scalerank/water.csv +++ b/spreadsheets/scalerank/water.csv @@ -1 +1 @@ -kind,scalerank +kind,output:scalerank::int diff --git a/spreadsheets/sort_key/boundaries.csv b/spreadsheets/sort_key/boundaries.csv index 32961158d..474e6c961 100644 --- a/spreadsheets/sort_key/boundaries.csv +++ b/spreadsheets/sort_key/boundaries.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int municipality,252 county,254 Admin-1 boundary,256 diff --git a/spreadsheets/sort_key/buildings.csv b/spreadsheets/sort_key/buildings.csv index 83f956d71..ae67dd88e 100644 --- a/spreadsheets/sort_key/buildings.csv +++ b/spreadsheets/sort_key/buildings.csv @@ -1,2 +1,2 @@ -kind,sort_key +kind,output:sort_key::int *,475 diff --git a/spreadsheets/sort_key/earth.csv b/spreadsheets/sort_key/earth.csv index ec4ba55b5..3a0dd92d3 100644 --- a/spreadsheets/sort_key/earth.csv +++ b/spreadsheets/sort_key/earth.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int cliff,227 arete,228 earth,10 diff --git a/spreadsheets/sort_key/landuse.csv b/spreadsheets/sort_key/landuse.csv index b8b204544..69e1f90a6 100644 --- a/spreadsheets/sort_key/landuse.csv +++ b/spreadsheets/sort_key/landuse.csv @@ -1,4 +1,4 @@ -kind,geometrytype,sort_key +kind,geometrytype,output:sort_key::int fence,*,269 snow_fence,*,268 retaining_wall,*,267 diff --git a/spreadsheets/sort_key/roads.csv b/spreadsheets/sort_key/roads.csv index fe7006959..171af5bb9 100644 --- a/spreadsheets/sort_key/roads.csv +++ b/spreadsheets/sort_key/roads.csv @@ -1,4 +1,4 @@ -zoom::int,layer::int,kind,kind_detail,service,is_bridge,is_tunnel,sort_key +zoom::int,layer::int,kind,kind_detail,service,is_bridge,is_tunnel,output:sort_key::int >=15,7,*,*,*,*,*,447 >=15,6,*,*,*,*,*,447 >=15,5,*,*,*,*,*,447 diff --git a/spreadsheets/sort_key/transit.csv b/spreadsheets/sort_key/transit.csv index f669098f5..cde6ac07e 100644 --- a/spreadsheets/sort_key/transit.csv +++ b/spreadsheets/sort_key/transit.csv @@ -1,4 +1,4 @@ -kind,sort_key +kind,output:sort_key::int light_rail,418 tram,417 funicular,416 diff --git a/spreadsheets/sort_key/water.csv b/spreadsheets/sort_key/water.csv index 8cd309b4e..20255bfa9 100644 --- a/spreadsheets/sort_key/water.csv +++ b/spreadsheets/sort_key/water.csv @@ -1,4 +1,4 @@ -kind,boundary,sort_key +kind,boundary,output:sort_key::int river,,201 canal,,201 stream,,201 diff --git a/test/test_transform.py b/test/test_transform.py index c1880bb0c..96414e7a4 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -21,8 +21,8 @@ def _call_fut(self, props): match_result = self.matcher(shape, props, z) if match_result is None: return None - k, v = match_result - return float(v) + kvs = match_result + return float(kvs.get('scalerank')) def test_area_most_important(self): props = dict(area=1000000) @@ -275,14 +275,15 @@ def test_geometry_type(self): zoom = 16 sort_key_result = self.matcher(shape, props, zoom) self.assertIsNotNone(sort_key_result) - _, sort_key = sort_key_result - self.assertEquals(int(sort_key), 265) + sort_key = sort_key_result.get('sort_key') + self.assertEquals(type(sort_key), int) + self.assertEquals(sort_key, 265) shape = shapely.geometry.Polygon([(0, 0), (1, 1), (0, 1), (0, 0)]) sort_key_result = self.matcher(shape, props, zoom) self.assertIsNotNone(sort_key_result) - _, sort_key = sort_key_result - self.assertEquals(int(sort_key), 223) + sort_key = sort_key_result.get('sort_key') + self.assertEquals(sort_key, 223) class BuildingsUnifyTest(unittest.TestCase): diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index f520130c8..18b522605 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -3289,7 +3289,7 @@ def match(self, other): return other in self.values def __repr__(self): - return repr(self.value) + return repr(self.values) class _GreaterThanEqualMatcher(object): @@ -3353,10 +3353,22 @@ def _parse_kt(key_type): class CSVMatcher(object): def __init__(self, fh): - keys = None - types = [] + # collects together column information in the form of a tuple + # (is_target, key, typ) where is_target is a boolean which is true + # if the column is a target (i.e: output) column, or false if it is + # a column to be matched. key is a string key, and typ is a function + # which returns a type of that column given a string as input. + # typically this is int(), float(), etc... + cols = None + + # each row is a potential match, and rows keeps a tuple for each of + # (matchers, target values), where matchers is an array of matcher + # objects, one for each non-target column. rows = [] + # keep a set of static matchers to cut down the number of objects. + # these matchers are "constant" and don't take any input, so this is + # a safe operation. self.static_any = _AnyMatcher() self.static_none = _NoneMatcher() self.static_some = _SomeMatcher() @@ -3365,23 +3377,44 @@ def __init__(self, fh): # CSV - allow whitespace after the comma reader = csv.reader(fh, skipinitialspace=True) for row in reader: - if keys is None: - target_key = row.pop(-1) - keys = [] + if cols is None: + cols = [] for key_type in row: + # target (output) columns start with 'output:' + col_is_target = key_type.startswith('output:') + if col_is_target: + key_type = key_type[len('output:'):] + + # all columns can have an optional type after ::, + # e.g: key::int if it's supposed to be an integer. key, typ = _parse_kt(key_type) - keys.append(key) - types.append(typ) + cols.append((col_is_target, key, typ)) else: - target_val = row.pop(-1) + # match_row is the array of matcher objects which will be + # called with each column's value. + match_row = [] + # target_vals is the dict of values to return to the caller + # for the first row where all the matchers return true. + target_vals = {} + for i in range(0, len(row)): - row[i] = self._match_val(row[i], types[i]) - rows.append((row, target_val)) + is_target, key, typ = cols[i] + val = row[i] + + if is_target: + target_vals[key] = typ(val) - self.keys = keys + else: + match_row.append(self._match_val(val, typ)) + + rows.append((match_row, target_vals)) + + # keys is the string key for each matcher, so that values can be + # extracted from the object being matched in the same order as the + # array of matcher objects. + self.keys = [key for is_target, key, typ in cols] self.rows = rows - self.target_key = target_key def _match_val(self, v, typ): if v == '*': @@ -3422,9 +3455,10 @@ def __call__(self, shape, properties, zoom): else: val = properties.get(key) vals.append(val) - for row, target_val in self.rows: + for row, target_vals in self.rows: + import sys if all([a.match(b) for (a, b) in zip(row, vals)]): - return (self.target_key, target_val) + return target_vals return None @@ -3440,7 +3474,6 @@ def csv_match_properties(ctx): source_layer = ctx.params.get('source_layer') start_zoom = ctx.params.get('start_zoom', 0) end_zoom = ctx.params.get('end_zoom') - target_value_type = ctx.params.get('target_value_type') matcher = ctx.resources.get('matcher') assert source_layer, 'csv_match_properties: missing source layer' @@ -3456,16 +3489,10 @@ def csv_match_properties(ctx): if layer is None: return None - def _type_cast(v): - if target_value_type == 'int': - return int(v) - return v - for shape, props, fid in layer['features']: m = matcher(shape, props, zoom) if m is not None: - k, v = m - props[k] = _type_cast(v) + props.update(m) return layer