Add wof l10n names

data/migrations/update-wof-l10n.py

@@ -0,0 +1,51 @@
+# This expects a local checkout of whos on first data. It will fetch
+# the data from the on disk location for all existing neighbourhoods
+# in the database, and generate updates for all neighbourhoods that
+# have other local names.
+
+from tilequeue.wof import make_wof_filesystem_neighbourhood_fetcher
+from tilequeue.wof import make_wof_model
+import os
+import sys
+import yaml
+
+cfg_path = '/etc/tilequeue/config.yaml'
+wof_path = '/var/whosonfirst-data'
+
+if not os.path.exists(cfg_path):
+    print 'No tilequeue config found. Not updating wof l10n.'
+    sys.exit(0)
+
+if not os.path.exists(wof_path):
+    print 'No woftilequeue data found. Not updating wof l10n.'
+    sys.exit(1)
+
+with open(cfg_path) as fh:
+    yaml_data = yaml.load(fh)
+
+wof_cfg = yaml_data['wof']
+psql_cfg = wof_cfg['postgresql']
+
+wof_model = make_wof_model(psql_cfg)
+metas = wof_model.find_previous_neighbourhood_meta()
+n_threads = 50
+fs_neighbourhood_fetcher = make_wof_filesystem_neighbourhood_fetcher(
+    wof_path, n_threads)
+ns, failures = fs_neighbourhood_fetcher.fetch_raw_neighbourhoods(metas)
+if failures:
+    print 'Errors fetching neighbourhoods'
+    # if there are more than 10, the rest will probably have failed
+    # for the same reason
+    for failure in failures[:10]:
+        print 'Failed fetching %d: %r - %r' % (
+            failure.wof_id, failure.reason, failure.message_one_line)
+    sys.exit(1)
+
+ns_to_update = []
+for n in ns:
+    if n.l10n_names:
+        ns_to_update.append(n)
+
+ns_to_add = []
+ids_to_remove = []
+wof_model.sync_neighbourhoods(ns_to_add, ns_to_update, ids_to_remove)

data/migrations/v1.0.0-schema-prefunction.sql

@@ -0,0 +1,13 @@
+DO $$
+BEGIN
+
+IF NOT EXISTS (
+  SELECT 1 FROM pg_attribute
+    WHERE attrelid = 'public.wof_neighbourhood'::regclass
+    AND attname = 'l10n_name'
+    AND NOT attisdropped) THEN
+
+  ALTER TABLE wof_neighbourhood ADD COLUMN l10n_name HSTORE;
+END IF;
+
+END $$;

data/schema.sql

@@ -21,11 +21,12 @@ CREATE TABLE wof_neighbourhood (
   min_zoom SMALLINT NOT NULL,
   max_zoom SMALLINT NOT NULL,
   is_landuse_aoi BOOLEAN,
+  label_position geometry(Point, 900913) NOT NULL,
+  geometry geometry(Geometry, 900913) NOT NULL,
   inception DATE NOT NULL DEFAULT '0001-01-01',
   cessation DATE NOT NULL DEFAULT '9999-12-31',
   is_visible BOOLEAN NOT NULL DEFAULT true,
-  label_position geometry(Point, 900913) NOT NULL,
-  geometry geometry(Geometry, 900913) NOT NULL
+  l10n_name HSTORE
 );
 
 CREATE INDEX wof_neighbourhood_label_position_index ON wof_neighbourhood USING GIST(label_position);

queries/places.jinja2

@@ -73,7 +73,7 @@ SELECT
     min_zoom,
     max_zoom,
     CASE WHEN is_landuse_aoi = true THEN true ELSE NULL END AS is_landuse_aoi,
-    NULL AS tags
+    %#l10n_name AS tags
 
 FROM wof_neighbourhood wof_n
 

test/418-wof-l10n_name.py

@@ -0,0 +1,42 @@
+# Hollywood (wof neighbourhood)
+# https://whosonfirst.mapzen.com/data/858/260/37/85826037.geojson
+assert_has_feature(
+    16, 11227, 26157, 'places',
+    { 'id': 85826037, 'kind': 'neighbourhood',
+      'source': "whosonfirst.mapzen.com",
+      'name': 'Hollywood',
+      'name:kor': '\xed\x97\x90\xeb\xa6\xac\xec\x9a\xb0\xeb\x93\x9c' })
+
+# San Francisco (wof neighbourhood)
+# https://whosonfirst.mapzen.com/data/858/826/41/85882641.geojson
+assert_has_feature(
+    16, 14893, 29234, 'places',
+    { 'id': 85882641, 'kind': 'neighbourhood',
+      'source': "whosonfirst.mapzen.com",
+      'name': 'San Francisco',
+      'name:spa': type(None) })
+
+# San Francisco (osm city)
+# https://whosonfirst.mapzen.com/data/858/826/41/85882641.geojson
+assert_has_feature(
+    16, 10482, 25330, 'places',
+    { 'id': 26819236, 'kind': 'city',
+      'source': "openstreetmap.org",
+      'name': 'San Francisco',
+      'name:zho': '\xe8\x88\x8a\xe9\x87\x91\xe5\xb1\xb1\xe5\xb8\x82\xe8\x88\x87\xe7\xb8\xa3' })
+
+# Node: Londonderry/Derry (267762522)
+# http://www.openstreetmap.org/node/267762522
+assert_has_feature(
+    16, 31436, 20731, 'places',
+    { 'id': 267762522, 'name:eng_GB': 'Londonderry'})
+
+# Node: Jerusalem (29090735)
+# http://www.openstreetmap.org/node/29090735
+assert_has_feature(
+    16, 39180, 26661, 'places',
+    { 'id': 29090735,
+      'name:nan': 'I\xc3\xa2-l\xc5\x8d\xcd\x98-sat-l\xc3\xa9ng',
+      'name:zho': '\xe8\x80\xb6\xe8\xb7\xaf\xe6\x92\x92\xe5\x86\xb7',
+      'name:yue': '\xe8\x80\xb6\xe8\xb7\xaf\xe6\x92\x92\xe5\x86\xb7',
+      })