diff --git a/data/migrations/update-wof-l10n.py b/data/migrations/update-wof-l10n.py new file mode 100644 index 000000000..b91ebd08e --- /dev/null +++ b/data/migrations/update-wof-l10n.py @@ -0,0 +1,51 @@ +# This expects a local checkout of whos on first data. It will fetch +# the data from the on disk location for all existing neighbourhoods +# in the database, and generate updates for all neighbourhoods that +# have other local names. + +from tilequeue.wof import make_wof_filesystem_neighbourhood_fetcher +from tilequeue.wof import make_wof_model +import os +import sys +import yaml + +cfg_path = '/etc/tilequeue/config.yaml' +wof_path = '/var/whosonfirst-data' + +if not os.path.exists(cfg_path): + print 'No tilequeue config found. Not updating wof l10n.' + sys.exit(0) + +if not os.path.exists(wof_path): + print 'No woftilequeue data found. Not updating wof l10n.' + sys.exit(1) + +with open(cfg_path) as fh: + yaml_data = yaml.load(fh) + +wof_cfg = yaml_data['wof'] +psql_cfg = wof_cfg['postgresql'] + +wof_model = make_wof_model(psql_cfg) +metas = wof_model.find_previous_neighbourhood_meta() +n_threads = 50 +fs_neighbourhood_fetcher = make_wof_filesystem_neighbourhood_fetcher( + wof_path, n_threads) +ns, failures = fs_neighbourhood_fetcher.fetch_raw_neighbourhoods(metas) +if failures: + print 'Errors fetching neighbourhoods' + # if there are more than 10, the rest will probably have failed + # for the same reason + for failure in failures[:10]: + print 'Failed fetching %d: %r - %r' % ( + failure.wof_id, failure.reason, failure.message_one_line) + sys.exit(1) + +ns_to_update = [] +for n in ns: + if n.l10n_names: + ns_to_update.append(n) + +ns_to_add = [] +ids_to_remove = [] +wof_model.sync_neighbourhoods(ns_to_add, ns_to_update, ids_to_remove) diff --git a/data/migrations/v1.0.0-schema-prefunction.sql b/data/migrations/v1.0.0-schema-prefunction.sql new file mode 100644 index 000000000..587c7a763 --- /dev/null +++ b/data/migrations/v1.0.0-schema-prefunction.sql @@ -0,0 +1,13 @@ +DO $$ +BEGIN + +IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = 'public.wof_neighbourhood'::regclass + AND attname = 'l10n_name' + AND NOT attisdropped) THEN + + ALTER TABLE wof_neighbourhood ADD COLUMN l10n_name HSTORE; +END IF; + +END $$; diff --git a/data/schema.sql b/data/schema.sql index 5105a99fc..fc1f50d8d 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -21,11 +21,12 @@ CREATE TABLE wof_neighbourhood ( min_zoom SMALLINT NOT NULL, max_zoom SMALLINT NOT NULL, is_landuse_aoi BOOLEAN, + label_position geometry(Point, 900913) NOT NULL, + geometry geometry(Geometry, 900913) NOT NULL, inception DATE NOT NULL DEFAULT '0001-01-01', cessation DATE NOT NULL DEFAULT '9999-12-31', is_visible BOOLEAN NOT NULL DEFAULT true, - label_position geometry(Point, 900913) NOT NULL, - geometry geometry(Geometry, 900913) NOT NULL + l10n_name HSTORE ); CREATE INDEX wof_neighbourhood_label_position_index ON wof_neighbourhood USING GIST(label_position); diff --git a/queries/places.jinja2 b/queries/places.jinja2 index 80a0c0ed5..f1c626e56 100644 --- a/queries/places.jinja2 +++ b/queries/places.jinja2 @@ -73,7 +73,7 @@ SELECT min_zoom, max_zoom, CASE WHEN is_landuse_aoi = true THEN true ELSE NULL END AS is_landuse_aoi, - NULL AS tags + %#l10n_name AS tags FROM wof_neighbourhood wof_n diff --git a/test/418-wof-l10n_name.py b/test/418-wof-l10n_name.py new file mode 100644 index 000000000..1e0f58f07 --- /dev/null +++ b/test/418-wof-l10n_name.py @@ -0,0 +1,42 @@ +# Hollywood (wof neighbourhood) +# https://whosonfirst.mapzen.com/data/858/260/37/85826037.geojson +assert_has_feature( + 16, 11227, 26157, 'places', + { 'id': 85826037, 'kind': 'neighbourhood', + 'source': "whosonfirst.mapzen.com", + 'name': 'Hollywood', + 'name:kor': '\xed\x97\x90\xeb\xa6\xac\xec\x9a\xb0\xeb\x93\x9c' }) + +# San Francisco (wof neighbourhood) +# https://whosonfirst.mapzen.com/data/858/826/41/85882641.geojson +assert_has_feature( + 16, 14893, 29234, 'places', + { 'id': 85882641, 'kind': 'neighbourhood', + 'source': "whosonfirst.mapzen.com", + 'name': 'San Francisco', + 'name:spa': type(None) }) + +# San Francisco (osm city) +# https://whosonfirst.mapzen.com/data/858/826/41/85882641.geojson +assert_has_feature( + 16, 10482, 25330, 'places', + { 'id': 26819236, 'kind': 'city', + 'source': "openstreetmap.org", + 'name': 'San Francisco', + 'name:zho': '\xe8\x88\x8a\xe9\x87\x91\xe5\xb1\xb1\xe5\xb8\x82\xe8\x88\x87\xe7\xb8\xa3' }) + +# Node: Londonderry/Derry (267762522) +# http://www.openstreetmap.org/node/267762522 +assert_has_feature( + 16, 31436, 20731, 'places', + { 'id': 267762522, 'name:eng_GB': 'Londonderry'}) + +# Node: Jerusalem (29090735) +# http://www.openstreetmap.org/node/29090735 +assert_has_feature( + 16, 39180, 26661, 'places', + { 'id': 29090735, + 'name:nan': 'I\xc3\xa2-l\xc5\x8d\xcd\x98-sat-l\xc3\xa9ng', + 'name:zho': '\xe8\x80\xb6\xe8\xb7\xaf\xe6\x92\x92\xe5\x86\xb7', + 'name:yue': '\xe8\x80\xb6\xe8\xb7\xaf\xe6\x92\x92\xe5\x86\xb7', + })