From 2e9b1291235f4e87e15938a34eeec75302f8bc0c Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Thu, 1 Sep 2016 18:09:22 -0400 Subject: [PATCH 1/9] Add a test for localized names in a tile with NY/NJ boundaries --- integration-test/1016-missing-localized-names.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 integration-test/1016-missing-localized-names.py diff --git a/integration-test/1016-missing-localized-names.py b/integration-test/1016-missing-localized-names.py new file mode 100644 index 000000000..eb2ba9ad8 --- /dev/null +++ b/integration-test/1016-missing-localized-names.py @@ -0,0 +1,10 @@ +# New Jersey - New York state boundary +# http://www.openstreetmap.org/relation/224951 (New Jersey) +# http://www.openstreetmap.org/relation/61320 (New York state) +assert_has_feature( + 15, 9643, 12327, "boundaries", + {"kind": "state", "name": "New York - New Jersey", + "name:right": "New York", "name:left": "New Jersey", + "name:right:es": "Nueva York", "name:left:es": "Nueva Jersey", + "name:right:lv":"\u0145\u016bd\u017eersija", "name:left:lv":"\u0145ujorka" + }) From 39797ff50dd1b3e67b805521734d0770bdb3ff5e Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Thu, 1 Sep 2016 19:01:17 -0400 Subject: [PATCH 2/9] Fix test framework to handle unicode expected-values properly --- integration-test.py | 3 +++ integration-test/1016-missing-localized-names.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/integration-test.py b/integration-test.py index 3e0b5cd72..7b7986f01 100644 --- a/integration-test.py +++ b/integration-test.py @@ -103,6 +103,9 @@ def match_properties(actual, expected): if not exp_v(v): return False + elif isinstance(exp_v, unicode): + return v == exp_v.encode('utf-8') + elif v != exp_v: return False diff --git a/integration-test/1016-missing-localized-names.py b/integration-test/1016-missing-localized-names.py index eb2ba9ad8..4a067b666 100644 --- a/integration-test/1016-missing-localized-names.py +++ b/integration-test/1016-missing-localized-names.py @@ -1,10 +1,12 @@ +# -*- coding: utf-8 -*- + # New Jersey - New York state boundary # http://www.openstreetmap.org/relation/224951 (New Jersey) # http://www.openstreetmap.org/relation/61320 (New York state) assert_has_feature( 15, 9643, 12327, "boundaries", - {"kind": "state", "name": "New York - New Jersey", - "name:right": "New York", "name:left": "New Jersey", + {"kind": "region", "name": "New Jersey - New York", + "name:right": "New York", "name:left": "New Jersey", "name:right:es": "Nueva York", "name:left:es": "Nueva Jersey", - "name:right:lv":"\u0145\u016bd\u017eersija", "name:left:lv":"\u0145ujorka" + "name:right:lv":u"Ņujorka", "name:left:lv":u"Ņūdžersija" }) From 3f01980f6cb26c64bbccb00e2c473b55153a07ea Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Thu, 1 Sep 2016 19:01:33 -0400 Subject: [PATCH 3/9] Add a source column to the boundaries queries --- queries/boundaries.jinja2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/queries/boundaries.jinja2 b/queries/boundaries.jinja2 index 36b01d768..1d5e560e5 100644 --- a/queries/boundaries.jinja2 +++ b/queries/boundaries.jinja2 @@ -60,6 +60,7 @@ WHERE SELECT tags->'name' AS name, + 'openstreetmap.org' AS source, NULL AS maritime_boundary, mz_calculate_json_boundaries(planet_osm_polygon.*) AS mz_properties, -- note that we force the RHR, which makes outers clockwise, @@ -84,6 +85,7 @@ UNION ALL SELECT NULL AS name, + 'openstreetmap.org' AS source, true AS maritime_boundary, '{"kind": "maritime"}'::json as mz_properties, {% filter geometry %}{{ bounds['polygon']|bbox_intersection('the_geom',3857) }}{% endfilter %} AS __geometry__, @@ -99,6 +101,7 @@ UNION ALL SELECT tags->'name' AS name, + 'openstreetmap.org' AS source, NULL AS maritime_boundary, mz_calculate_json_boundaries(planet_osm_line.*) AS mz_properties, {% filter geometry %}way{% endfilter %} AS __geometry__, From c188fa32cbcdaf2ca905b6d7c146d38726ef88af Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Tue, 6 Sep 2016 16:04:38 -0400 Subject: [PATCH 4/9] Specify naturalearthdata as a source for boundaries --- queries/boundaries.jinja2 | 1 + 1 file changed, 1 insertion(+) diff --git a/queries/boundaries.jinja2 b/queries/boundaries.jinja2 index 1d5e560e5..dc11f43ff 100644 --- a/queries/boundaries.jinja2 +++ b/queries/boundaries.jinja2 @@ -1,5 +1,6 @@ {% macro ne_boundary_cols() %} gid AS __id__, + 'naturalearthdata.com' AS source, {% filter geometry %}the_geom{% endfilter %} AS __geometry__, mz_calculate_json_boundaries(t.*) AS mz_properties {% endmacro %} From e36f4f781908a72a021017cebb002a1505a849b4 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Wed, 7 Sep 2016 10:04:18 -0400 Subject: [PATCH 5/9] Flip New York and New Jersey to fix tests --- integration-test/1016-missing-localized-names.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-test/1016-missing-localized-names.py b/integration-test/1016-missing-localized-names.py index 4a067b666..30e9c91f3 100644 --- a/integration-test/1016-missing-localized-names.py +++ b/integration-test/1016-missing-localized-names.py @@ -5,8 +5,8 @@ # http://www.openstreetmap.org/relation/61320 (New York state) assert_has_feature( 15, 9643, 12327, "boundaries", - {"kind": "region", "name": "New Jersey - New York", - "name:right": "New York", "name:left": "New Jersey", - "name:right:es": "Nueva York", "name:left:es": "Nueva Jersey", - "name:right:lv":u"Ņujorka", "name:left:lv":u"Ņūdžersija" + {"kind": "region", "name": "New York - New Jersey", + "name:left": "New York", "name:right": "New Jersey", + "name:left:es": "Nueva York", "name:right:es": "Nueva Jersey", + "name:left:lv":u"Ņujorka", "name:right:lv":u"Ņūdžersija" }) From 6fc386c33762330fbdd4d7882b46e81ca3d90575 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Wed, 7 Sep 2016 12:03:02 -0400 Subject: [PATCH 6/9] Normalize unicode expected values to utf8-encoded bytestrings, too --- integration-test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration-test.py b/integration-test.py index 7b7986f01..3637baede 100644 --- a/integration-test.py +++ b/integration-test.py @@ -133,6 +133,8 @@ def match_distance(actual, expected): # normalise unicode values if isinstance(v, unicode): v = v.encode('utf-8') + if isinstance(exp_v, unicode): + exp_v = exp_v.encode('utf-8') if exp_v is not None: if isinstance(exp_v, set): From 364331b75e6f5463a6cfe68d0dcdf6b4882dfd96 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Wed, 7 Sep 2016 14:35:11 -0400 Subject: [PATCH 7/9] Sort the features on a layer in the admin_boundaries transform method --- vectordatasource/transform.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 12a522304..fe5996fdb 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -1752,6 +1752,10 @@ def admin_boundaries(ctx): maritime_features = list() new_features = list() + # Sorting here so that we have consistent ordering of left/right side + # on boundaries. + layer['features'] = sorted(layer['features'], key=lambda f: f[1]['id']) + for shape, props, fid in layer['features']: dims = _geom_dimensions(shape) kind = props.get('kind') From 9ce573a5952525edac658310a0cba178f604d494 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Wed, 7 Sep 2016 16:31:40 -0400 Subject: [PATCH 8/9] Sort and adjust test --- integration-test/1016-missing-localized-names.py | 8 ++++---- vectordatasource/transform.py | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/integration-test/1016-missing-localized-names.py b/integration-test/1016-missing-localized-names.py index 30e9c91f3..4a067b666 100644 --- a/integration-test/1016-missing-localized-names.py +++ b/integration-test/1016-missing-localized-names.py @@ -5,8 +5,8 @@ # http://www.openstreetmap.org/relation/61320 (New York state) assert_has_feature( 15, 9643, 12327, "boundaries", - {"kind": "region", "name": "New York - New Jersey", - "name:left": "New York", "name:right": "New Jersey", - "name:left:es": "Nueva York", "name:right:es": "Nueva Jersey", - "name:left:lv":u"Ņujorka", "name:right:lv":u"Ņūdžersija" + {"kind": "region", "name": "New Jersey - New York", + "name:right": "New York", "name:left": "New Jersey", + "name:right:es": "Nueva York", "name:left:es": "Nueva Jersey", + "name:right:lv":u"Ņujorka", "name:left:lv":u"Ņūdžersija" }) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index fe5996fdb..788bd0f41 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -1754,9 +1754,7 @@ def admin_boundaries(ctx): # Sorting here so that we have consistent ordering of left/right side # on boundaries. - layer['features'] = sorted(layer['features'], key=lambda f: f[1]['id']) - - for shape, props, fid in layer['features']: + for shape, props, fid in sorted(layer['features'], key=lambda f: f[1]['id']): dims = _geom_dimensions(shape) kind = props.get('kind') maritime_boundary = props.get('maritime_boundary') From d9bd72fa203df49ea8d71435ec199f69a2789d4d Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Wed, 7 Sep 2016 17:03:10 -0400 Subject: [PATCH 9/9] Appease line-too-long error --- vectordatasource/transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 788bd0f41..68e82870f 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -1754,7 +1754,9 @@ def admin_boundaries(ctx): # Sorting here so that we have consistent ordering of left/right side # on boundaries. - for shape, props, fid in sorted(layer['features'], key=lambda f: f[1]['id']): + sorted_layer = sorted(layer['features'], key=lambda f: f[1]['id']) + + for shape, props, fid in sorted_layer: dims = _geom_dimensions(shape) kind = props.get('kind') maritime_boundary = props.get('maritime_boundary')