From 18f4eabf8a291baafd38958c60ce003a519ddafe Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Tue, 26 Jul 2022 10:57:26 -0400 Subject: [PATCH 01/12] add support for preprocessed inline geojson layers --- tilequeue/command.py | 1 + tilequeue/process.py | 55 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index ae66e97..371ffc0 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -544,6 +544,7 @@ def parse_layer_data(query_cfg, buffer_cfg, cfg_path): query_bounds_pad_fn=create_query_bounds_pad_fn( buffer_cfg, layer_name), tolerance=float(layer_config.get('tolerance', 1.0)), + layer_path=layer_config.get('layer_path'), ) layer_data.append(layer_datum) if layer_name in all_layer_names: diff --git a/tilequeue/process.py b/tilequeue/process.py index 480143b..69d6d15 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -1,12 +1,15 @@ from __future__ import division +import itertools +import json +import os.path from collections import defaultdict from collections import namedtuple from cStringIO import StringIO from sys import getsizeof from shapely import geometry -from shapely.geometry import MultiPolygon +from shapely.geometry import MultiPolygon, shape, GeometryCollection from shapely.wkb import loads from zope.dottedname.resolve import resolve @@ -69,7 +72,6 @@ def _sizeof(val): 'log', ]) - # post-process all the layers simultaneously, which allows new # layers to be created from processing existing ones (e.g: for # computed centroids) or modifying layers based on the contents @@ -89,6 +91,7 @@ def _log_fn(data): if log_fn: log_fn(dict(fn_name=step['fn_name'], msg=data)) + # create a context ctx = Context( feature_layers=feature_layers, nominal_zoom=nominal_zoom, @@ -98,6 +101,7 @@ def _log_fn(data): log=_log_fn, ) + # apply the actual function layer = fn(ctx) feature_layers = ctx.feature_layers if layer is not None: @@ -146,7 +150,6 @@ def _cut_coord( padded_bounds=padded_bounds, ) cut_feature_layers.append(cut_feature_layer) - return cut_feature_layers @@ -286,6 +289,12 @@ def process_coord_no_format( # filter, and then transform each layer as necessary for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] + # inline layers are expected to be pre-processed + layer_path = layer_datum.get('layer_path') + if layer_path is not None: + processed_feature_layers.append(feature_layer) + continue + layer_name = layer_datum['name'] geometry_types = layer_datum['geometry_types'] padded_bounds = feature_layer['padded_bounds'] @@ -512,6 +521,37 @@ def _calculate_scale(scale, coord, nominal_zoom): return scale +def _load_inline_layer(layer_path): + # we can optionally load geojson layers from file all we need is a file path per layer name. we expect the coords to + # already be in mercator projection and we only fill out the minimal structure of the feature tuple, ie there's no ID, + # most of the datum stuff is missing. we are mostly worried about the shape and properties and we skip any layers + # that are configured but cannot be loaded for whatever reason + features = [] + + # has to exist and has to be geojson for now + if not os.path.isfile(layer_path) or not layer_path.endswith('.geojson'): + return features + + # load the geojson into a shapely geometry collection + with open(layer_path) as fh: + try: + fc = json.load(fh) + # skip if this isnt pseudo mercator + if fc['crs']['properties']['name'] != 'urn:ogc:def:crs:EPSG::3857': + return features + gc = GeometryCollection([shape(feature['geometry']) for feature in fc['features']]) + except: + return features + + # add the features geometries with their properties (values must be strings) in tuples + for geom, feat in itertools.izip(gc.geoms, fc['features']): + properties = {k: str(v) if type(v) is not bool else str(v).lower() for k, v in + feat['properties'].iteritems()} + features.append((geom, properties, None)) + + return features + + def format_coord( coord, nominal_zoom, max_zoom_with_changes, processed_feature_layers, formats, unpadded_bounds, cut_coords, buffer_cfg, extra_data, scale): @@ -645,7 +685,7 @@ def convert_source_data_to_feature_layers(rows, layer_data, unpadded_bounds, zoo # expecting downstream in the process_coord function for layer_datum in layer_data: layer_name = layer_datum['name'] - layer_props = row_props_by_layer[layer_name] + layer_props = row_props_by_layer.get(layer_name) if layer_props is not None: props = common_props.copy() props.update(layer_props) @@ -665,6 +705,13 @@ def convert_source_data_to_feature_layers(rows, layer_data, unpadded_bounds, zoo features_by_layer[layer_name].append(query_props) + # inline layers contain a path to their features inline in their datum + for layer_datum in layer_data: + layer_name = layer_datum['name'] + layer_path = layer_datum.get('layer_path') + if layer_path is not None: + features_by_layer[layer_name].extend(_load_inline_layer(layer_path)) + feature_layers = [] for layer_datum in layer_data: layer_name = layer_datum['name'] From 13c9d515570d3fa9835550dfb986be788b54372f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:05:22 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tilequeue/process.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tilequeue/process.py b/tilequeue/process.py index 69d6d15..0226f9a 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -9,7 +9,9 @@ from sys import getsizeof from shapely import geometry -from shapely.geometry import MultiPolygon, shape, GeometryCollection +from shapely.geometry import GeometryCollection +from shapely.geometry import MultiPolygon +from shapely.geometry import shape from shapely.wkb import loads from zope.dottedname.resolve import resolve @@ -77,6 +79,8 @@ def _sizeof(val): # computed centroids) or modifying layers based on the contents # of other layers (e.g: projecting attributes, deleting hidden # features, etc...) + + def _postprocess_data( feature_layers, post_process_data, nominal_zoom, unpadded_bounds, log_fn=None): From 4ad7535d6cfbfe49b1f20b83c2aec0d738dc6ca2 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Tue, 26 Jul 2022 11:32:22 -0400 Subject: [PATCH 03/12] raise when you said it would be there and it is but we cant use it. in this case you intended it to be used --- tilequeue/command.py | 1 + tilequeue/process.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index 371ffc0..36b00bb 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2177,6 +2177,7 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) + pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # TODO: testing hack remove coord_data = [dict(coord=x) for x in pyramid_coords] try: diff --git a/tilequeue/process.py b/tilequeue/process.py index 69d6d15..7934152 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -522,10 +522,10 @@ def _calculate_scale(scale, coord, nominal_zoom): def _load_inline_layer(layer_path): - # we can optionally load geojson layers from file all we need is a file path per layer name. we expect the coords to - # already be in mercator projection and we only fill out the minimal structure of the feature tuple, ie there's no ID, - # most of the datum stuff is missing. we are mostly worried about the shape and properties and we skip any layers - # that are configured but cannot be loaded for whatever reason + # we can optionally load geojson layers from file all we need is a file path. we expect the coordinates to already + # be in mercator projection and we only fill out the minimal structure of the feature tuple, ie there's no ID. if + # the path is given and the file exists, we expect to be able to load the layer's features. so if the json is + # malformed or shapely can't make sense of the geom then we'll raise features = [] # has to exist and has to be geojson for now @@ -534,14 +534,12 @@ def _load_inline_layer(layer_path): # load the geojson into a shapely geometry collection with open(layer_path) as fh: - try: - fc = json.load(fh) - # skip if this isnt pseudo mercator - if fc['crs']['properties']['name'] != 'urn:ogc:def:crs:EPSG::3857': - return features - gc = GeometryCollection([shape(feature['geometry']) for feature in fc['features']]) - except: + fc = json.load(fh) + # skip if this isnt pseudo mercator + if fc['crs']['properties']['name'] != 'urn:ogc:def:crs:EPSG::3857': return features + gc = GeometryCollection([shape(feature['geometry']) for feature in fc['features']]) + # add the features geometries with their properties (values must be strings) in tuples for geom, feat in itertools.izip(gc.geoms, fc['features']): From 929fc983dcdd00bc4033580a1790ab290deee518 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:33:21 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tilequeue/command.py | 2 +- tilequeue/process.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index 36b00bb..2ab102a 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2177,7 +2177,7 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) - pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # TODO: testing hack remove + pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # TODO: testing hack remove coord_data = [dict(coord=x) for x in pyramid_coords] try: diff --git a/tilequeue/process.py b/tilequeue/process.py index 7d5827e..891a599 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -544,7 +544,6 @@ def _load_inline_layer(layer_path): return features gc = GeometryCollection([shape(feature['geometry']) for feature in fc['features']]) - # add the features geometries with their properties (values must be strings) in tuples for geom, feat in itertools.izip(gc.geoms, fc['features']): properties = {k: str(v) if type(v) is not bool else str(v).lower() for k, v in From c2461f55835c69d29d24239030b7779cd4c81266 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Tue, 26 Jul 2022 14:26:07 -0400 Subject: [PATCH 05/12] remove testing stuff --- tilequeue/command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index 2ab102a..371ffc0 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2177,7 +2177,6 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) - pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # TODO: testing hack remove coord_data = [dict(coord=x) for x in pyramid_coords] try: From a05045171f3489b87e9892b50f4119d4dc40ff58 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Tue, 26 Jul 2022 14:27:05 -0400 Subject: [PATCH 06/12] whitespace revert --- tilequeue/process.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tilequeue/process.py b/tilequeue/process.py index 891a599..d5ca6d5 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -79,8 +79,6 @@ def _sizeof(val): # computed centroids) or modifying layers based on the contents # of other layers (e.g: projecting attributes, deleting hidden # features, etc...) - - def _postprocess_data( feature_layers, post_process_data, nominal_zoom, unpadded_bounds, log_fn=None): @@ -95,7 +93,6 @@ def _log_fn(data): if log_fn: log_fn(dict(fn_name=step['fn_name'], msg=data)) - # create a context ctx = Context( feature_layers=feature_layers, nominal_zoom=nominal_zoom, @@ -105,7 +102,6 @@ def _log_fn(data): log=_log_fn, ) - # apply the actual function layer = fn(ctx) feature_layers = ctx.feature_layers if layer is not None: @@ -120,7 +116,6 @@ def _log_fn(data): # append it. if layer is not None: feature_layers.append(layer) - return feature_layers From 68099752044faac13950b6b234dcfa8154229bfb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 18:27:20 +0000 Subject: [PATCH 07/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tilequeue/process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tilequeue/process.py b/tilequeue/process.py index d5ca6d5..a3562fa 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -79,6 +79,8 @@ def _sizeof(val): # computed centroids) or modifying layers based on the contents # of other layers (e.g: projecting attributes, deleting hidden # features, etc...) + + def _postprocess_data( feature_layers, post_process_data, nominal_zoom, unpadded_bounds, log_fn=None): From d2f4d431eeac400a86abfe8a6ff952a8d7318c48 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Wed, 27 Jul 2022 12:06:37 -0400 Subject: [PATCH 08/12] rename layer path config, dont stringify properties, raise if anything is wrong with the landmarks geojson --- tilequeue/command.py | 3 +-- tilequeue/process.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index 36b00bb..e2feba5 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -544,7 +544,7 @@ def parse_layer_data(query_cfg, buffer_cfg, cfg_path): query_bounds_pad_fn=create_query_bounds_pad_fn( buffer_cfg, layer_name), tolerance=float(layer_config.get('tolerance', 1.0)), - layer_path=layer_config.get('layer_path'), + pre_processed_layer_path=layer_config.get('pre_processed_layer_path'), ) layer_data.append(layer_datum) if layer_name in all_layer_names: @@ -2177,7 +2177,6 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) - pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # TODO: testing hack remove coord_data = [dict(coord=x) for x in pyramid_coords] try: diff --git a/tilequeue/process.py b/tilequeue/process.py index 7d5827e..8ffc630 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -294,7 +294,7 @@ def process_coord_no_format( for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] # inline layers are expected to be pre-processed - layer_path = layer_datum.get('layer_path') + layer_path = layer_datum.get('pre_processed_layer_path') if layer_path is not None: processed_feature_layers.append(feature_layer) continue @@ -532,8 +532,8 @@ def _load_inline_layer(layer_path): # malformed or shapely can't make sense of the geom then we'll raise features = [] - # has to exist and has to be geojson for now - if not os.path.isfile(layer_path) or not layer_path.endswith('.geojson'): + # has to exist + if not os.path.isfile(layer_path): return features # load the geojson into a shapely geometry collection @@ -541,15 +541,13 @@ def _load_inline_layer(layer_path): fc = json.load(fh) # skip if this isnt pseudo mercator if fc['crs']['properties']['name'] != 'urn:ogc:def:crs:EPSG::3857': - return features + raise Exception('Pre-processed layers must be in pseudo mercator projection') gc = GeometryCollection([shape(feature['geometry']) for feature in fc['features']]) - - # add the features geometries with their properties (values must be strings) in tuples + # add the features geometries with their properties in tuples for geom, feat in itertools.izip(gc.geoms, fc['features']): - properties = {k: str(v) if type(v) is not bool else str(v).lower() for k, v in - feat['properties'].iteritems()} - features.append((geom, properties, None)) + props = feat['properties'] + features.append((geom, props, props.get('id'))) return features @@ -710,7 +708,7 @@ def convert_source_data_to_feature_layers(rows, layer_data, unpadded_bounds, zoo # inline layers contain a path to their features inline in their datum for layer_datum in layer_data: layer_name = layer_datum['name'] - layer_path = layer_datum.get('layer_path') + layer_path = layer_datum.get('pre_processed_layer_path') if layer_path is not None: features_by_layer[layer_name].extend(_load_inline_layer(layer_path)) From 937fc29a68fc42de68aacbc0bdfba34212430178 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Wed, 27 Jul 2022 16:19:19 -0400 Subject: [PATCH 09/12] pre processed layers dont come from postgres and arent in rawr tiles --- tilequeue/query/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tilequeue/query/__init__.py b/tilequeue/query/__init__.py index 1230213..0cbc5ba 100644 --- a/tilequeue/query/__init__.py +++ b/tilequeue/query/__init__.py @@ -179,6 +179,9 @@ def _make_layer_info(layer_data, process_yaml_cfg): functions = _parse_yaml_functions(process_yaml_cfg) for layer_datum in layer_data: + # preprocessed layers are not from the database and not found in the rawr tile + if layer_datum.get('pre_processed_layer_path') is not None: + continue name = layer_datum['name'] min_zoom_fn, props_fn = functions[name] shape_types = ShapeType.parse_set(layer_datum['geometry_types']) From 40ba90a7da6295257f0ed64a23b595c43eb81fa8 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Thu, 28 Jul 2022 11:59:57 -0400 Subject: [PATCH 10/12] leave in testing commented out for future testers --- tilequeue/command.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tilequeue/command.py b/tilequeue/command.py index e2feba5..a3bc614 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2177,6 +2177,8 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) + # for brevity of testing it is sometimes convenient to reduce to a single child coord + #pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] coord_data = [dict(coord=x) for x in pyramid_coords] try: From 8e7752054a45b18e51d1061531fad390226a29bb Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Thu, 28 Jul 2022 14:10:21 -0400 Subject: [PATCH 11/12] lint --- tilequeue/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index a3bc614..5101544 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2178,7 +2178,7 @@ def tilequeue_meta_tile(cfg, args): pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) # for brevity of testing it is sometimes convenient to reduce to a single child coord - #pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] + # pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] coord_data = [dict(coord=x) for x in pyramid_coords] try: From 542e31022e9f3e9733a207f55bffea77164922f3 Mon Sep 17 00:00:00 2001 From: Kevin Kreiser Date: Mon, 1 Aug 2022 16:25:51 -0400 Subject: [PATCH 12/12] add a bit more information to the testing breadcrumb --- tilequeue/command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index 5101544..ebbe785 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -2177,8 +2177,9 @@ def tilequeue_meta_tile(cfg, args): # each coord here is the unit of work now pyramid_coords = [job_coord] pyramid_coords.extend(coord_children_range(job_coord, zoom_stop)) - # for brevity of testing it is sometimes convenient to reduce to a single child coord - # pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] + # for brevity of testing it is sometimes convenient to reduce to a single child coord that covers your test area + # this makes it so that only that metatile is built rather than the whole tile pyramid which can save 20-50min + # pyramid_coords = [Coordinate(zoom=13, column=2411, row=3080)] # this example covers liberty island at max zoom coord_data = [dict(coord=x) for x in pyramid_coords] try: