diff --git a/config.py b/config.py index 351e991..35f328d 100644 --- a/config.py +++ b/config.py @@ -38,3 +38,13 @@ 'application/x-protobuf', 'application/json', ] + +# Land cover layer is built using Tapalcatl2 archives that require a bit of extra configuration: +# The maximum zoom level for the land cover data is different than the vector tiles +LANDCOVER_MAX_ZOOM = 13 +# Tapalcatl2 archives are "materialized" at particular zoom levels at build time. +# These are the zooms we picked when building the land cover layer +LANDCOVER_MATERIALIZED_ZOOMS = [0, 7] +# Tapalcatl2 archives can contain multiple neighboring tiles to form a "metatile" +# THe land cover build used a metatile size of 1 +LANDCOVER_METATILE_SIZE = 1 diff --git a/server.py b/server.py index c528502..8723ef8 100644 --- a/server.py +++ b/server.py @@ -51,7 +51,7 @@ def setup_logging(): "mvtb": "application/x-protobuf", "topojson": "application/json", } -TileRequest = namedtuple('TileRequest', ['z', 'x', 'y', 'format']) +TileRequest = namedtuple('TileRequest', ['z', 'x', 'y', 'scale', 'format']) CacheInfo = namedtuple('CacheInfo', ['last_modified', 'etag']) StorageResponse = namedtuple('StorageResponse', ['data', 'cache_info']) @@ -101,7 +101,7 @@ def meta_and_offset(requested_tile, meta_size, tile_size, # zooms. this might change the effective delta between the zoom level of # the request and the zoom level of the metatile. if requested_tile.z < delta_z: - meta = TileRequest(0, 0, 0, 'zip') + meta = TileRequest(0, 0, 0, 1, 'zip') else: # allows setting a maximum detail level beyond which all features are @@ -120,6 +120,7 @@ def meta_and_offset(requested_tile, meta_size, tile_size, requested_tile.z - delta_z, requested_tile.x >> delta_z, requested_tile.y >> delta_z, + 1, 'zip', ) @@ -128,6 +129,7 @@ def meta_and_offset(requested_tile, meta_size, tile_size, actual_delta_z, requested_tile.x - (meta.x << actual_delta_z), requested_tile.y - (meta.y << actual_delta_z), + requested_tile.scale, requested_tile.format, ) @@ -254,7 +256,7 @@ def metatile_fetch(meta, cache_info): raise MetatileNotModifiedException() elif error_code == 'NoSuchKey': raise MetatileNotFoundException( - "No tile found at s3://%s/%s" % (s3_bucket, s3_key) + "No metatile found at s3://%s/%s" % (s3_bucket, s3_key) ) else: raise UnknownMetatileException( @@ -299,8 +301,8 @@ def retrieve_tile(meta, offset, cache_info): ) -def is_valid_tile_request(z, x, y): - return (0 <= z < 17) and (0 <= x < 2**z) and (0 <= y < 2**z) +def is_valid_tile_request(z, x, y, max_zoom=17): + return (0 <= z < max_zoom) and (0 <= x < 2**z) and (0 <= y < 2**z) @tile_bp.route('/tilezen/vector/v1//all///.') @@ -313,11 +315,10 @@ def handle_tile(z, x, y, fmt, tile_pixel_size=None): tile_size = tile_pixel_size / 256 if tile_size != int(tile_size): return abort(400, "Invalid tile size. %s is not a multiple of 256." % tile_pixel_size) - - requested_tile = TileRequest(z, x, y, fmt) - tile_size = int(tile_size) + requested_tile = TileRequest(z, x, y, tile_size, fmt) + meta, offset = meta_and_offset( requested_tile, current_app.config.get('METATILE_SIZE'), @@ -381,6 +382,112 @@ def tilejson(fmt, tile_pixel_size=None): return resp +def t2_meta_and_offset(requested_tile, materialized_zooms, metatile_size): + # Find the materialized zoom that holds this tile + mz = next(x for x in sorted(materialized_zooms, reverse=True) if x <= requested_tile.z) + + # Find the tile at the materialized zoom that holds the requested tile + dz = requested_tile.z - mz + mx = requested_tile.x >> dz + my = requested_tile.y >> dz + mx -= mx % metatile_size + my -= my % metatile_size + + meta = TileRequest(mz, mx, my, 1, 'zip') + + # Build the key for the tile inside the archive + offset = requested_tile + + return meta, offset + + +def t2_extract_tile(metatile_bytes, offset): + data = BytesIO(metatile_bytes) + z = zipfile.ZipFile(data, 'r') + + offset_key = '{zoom}/{x}/{y}{scale}.{format}'.format( + zoom=offset.z, + x=offset.x, + y=offset.y, + scale='' if offset.scale < 2 else '@%dx' % offset.scale, + format=offset.format, + ) + + try: + return z.read(offset_key) + except KeyError as e: + raise TileNotFoundInMetatile("Couldn't find tile %s in metatile" % offset_key) + + +def t2_retrieve_tile(meta, offset, cache_info): + metatile_data = metatile_fetch(meta, cache_info) + tile_data = t2_extract_tile(metatile_data.data, offset) + + return StorageResponse( + data=tile_data, + cache_info=CacheInfo( + last_modified=metatile_data.cache_info.last_modified, + etag=metatile_data.cache_info.etag, + ) + ) + + +@tile_bp.route('/tilezen/landcover/v1//all///.') +@tile_bp.route('/tilezen/landcover/v1/all///.') +def handle_landcover_tile(z, x, y, fmt, tile_pixel_size=None): + if not is_valid_tile_request(z, x, y, max_zoom=current_app.config.get('LANDCOVER_MAX_ZOOM')): + return abort(400, "Requested tile out of range.") + + tile_pixel_size = tile_pixel_size or 256 + tile_size = tile_pixel_size / 256 + if tile_size != int(tile_size): + return abort(400, "Invalid tile size. %s is not a multiple of 256." % tile_pixel_size) + if tile_size != 2: + return abort(400, "Landcover only supports 512 tile size.") + tile_size = int(tile_size) + + requested_tile = TileRequest(z, x, y, tile_size, fmt) + + t2_materialized_zooms = [0, 7] + t2_metatile_size = 1 + + (meta, offset) = t2_meta_and_offset( + requested_tile, + current_app.config.get('LANDCOVER_MATERIALIZED_ZOOMS'), + current_app.config.get('LANDCOVER_METATILE_SIZE'), + ) + + request_cache_info = CacheInfo( + last_modified=parse_header_time(request.headers.get('If-Modified-Since')), + etag=request.headers.get('If-None-Match'), + ) + + try: + storage_result = t2_retrieve_tile(meta, offset, request_cache_info) + + response = make_response(storage_result.data) + response.content_type = MIME_TYPES.get(fmt) + response.last_modified = storage_result.cache_info.last_modified + response.cache_control.public = True + response.cache_control.max_age = current_app.config.get("CACHE_MAX_AGE") + if current_app.config.get("SHARED_CACHE_MAX_AGE"): + response.cache_control.s_maxage = current_app.config.get("SHARED_CACHE_MAX_AGE") + response.set_etag(storage_result.cache_info.etag) + return response + + except MetatileNotFoundException: + current_app.logger.exception("Could not find metatile") + return "Metatile not found", 404 + except TileNotFoundInMetatile: + current_app.logger.exception("Could not find tile in metatile") + return "Tile not found", 404 + except MetatileNotModifiedException: + return "", 304 + except UnknownMetatileException: + current_app.logger.exception("Error fetching metatile") + return "Metatile fetch problem", 500 + + @tile_bp.route('/health_check') def health_check(): handle_tile(0, 0, 0, 'mvt', tile_pixel_size=256)