From f56be4d35506d1aa3d930f1c410c0640264cfe2b Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 18 Dec 2023 22:31:53 -0500 Subject: [PATCH 1/2] Multiple mbtiles and martin-cp fixes * BREAKING: `martin-cp` will now set `format=pbf` instead of `mvt`. This is what QGIS and possibly others expect, and this is what tools like tilelive generates. * `martin-cp` sets `minzoom` and `maxzoom` metadata values based on the zoom parameters * Add `mbtiles meta-update` command to refresh zoom levels based on the present tiles. --- Cargo.lock | 4 +- martin-tile-utils/Cargo.toml | 2 +- martin-tile-utils/src/lib.rs | 14 +++++ martin/src/bin/martin-cp.rs | 37 +++++++++----- mbtiles/Cargo.toml | 2 +- mbtiles/src/bin/mbtiles.rs | 10 ++++ mbtiles/src/lib.rs | 2 + mbtiles/src/update.rs | 24 +++++++++ mbtiles/tests/copy.rs | 10 ++++ .../tests/snapshots/copy__update@update.snap | 51 +++++++++++++++++++ .../martin-cp/flat-with-hash_metadata.txt | 4 +- tests/expected/martin-cp/flat_metadata.txt | 4 +- .../martin-cp/normalized_metadata.txt | 2 +- 13 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 mbtiles/src/update.rs create mode 100644 mbtiles/tests/snapshots/copy__update@update.snap diff --git a/Cargo.lock b/Cargo.lock index c56b8897f..576c712ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1893,7 +1893,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.3.0" +version = "0.3.1" dependencies = [ "approx", "insta", @@ -1901,7 +1901,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.8.4" +version = "0.8.5" dependencies = [ "actix-rt", "anyhow", diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 223f89c88..d03c158b4 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.3.0" +version = "0.3.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 5e7aa4f0f..029949ebe 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -35,6 +35,20 @@ impl Format { }) } + /// Get the `format` value as it should be stored in the `MBTiles` metadata table + #[must_use] + pub fn metadata_format_value(&self) -> &'static str { + match *self { + Self::Gif => "gif", + Self::Jpeg => "jpeg", + Self::Json => "json", + // QGIS uses `pbf` instead of `mvt` for some reason + Self::Mvt => "pbf", + Self::Png => "png", + Self::Webp => "webp", + } + } + #[must_use] pub fn content_type(&self) -> &str { match *self { diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 0334a986e..55ebe0928 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::fmt::{Debug, Display, Formatter}; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; @@ -153,20 +154,12 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { fn compute_tile_ranges(args: &CopyArgs) -> Vec { let mut ranges = Vec::new(); - let mut zooms_vec = Vec::new(); - let zooms = if let Some(max_zoom) = args.max_zoom { - let min_zoom = args.min_zoom.unwrap_or(0); - zooms_vec.extend(min_zoom..=max_zoom); - &zooms_vec - } else { - &args.zoom_levels - }; let boxes = if args.bbox.is_empty() { vec![Bounds::MAX_TILED] } else { args.bbox.clone() }; - for zoom in zooms { + for zoom in get_zooms(args).iter() { for bbox in &boxes { let (min_x, min_y, max_x, max_y) = bbox_to_xyz(bbox.left, bbox.bottom, bbox.right, bbox.top, *zoom); @@ -179,6 +172,17 @@ fn compute_tile_ranges(args: &CopyArgs) -> Vec { ranges } +fn get_zooms(args: &CopyArgs) -> Cow> { + if let Some(max_zoom) = args.max_zoom { + let mut zooms_vec = Vec::new(); + let min_zoom = args.min_zoom.unwrap_or(0); + zooms_vec.extend(min_zoom..=max_zoom); + Cow::Owned(zooms_vec) + } else { + Cow::Borrowed(&args.zoom_levels) + } +} + struct TileXyz { xyz: TileCoord, data: TileData, @@ -281,7 +285,7 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> } else { CopyDuplicateMode::Override }; - let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, args.mbt_type).await?; + let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?; let query = args.url_query.as_deref(); let req = TestRequest::default() .insert_header((ACCEPT_ENCODING, args.encoding.as_str())) @@ -371,10 +375,10 @@ async fn init_schema( conn: &mut SqliteConnection, sources: &[&dyn Source], tile_info: TileInfo, - mbt_type: Option, + args: &CopyArgs, ) -> Result { Ok(if is_empty_database(&mut *conn).await? { - let mbt_type = match mbt_type.unwrap_or(MbtTypeCli::Normalized) { + let mbt_type = match args.mbt_type.unwrap_or(MbtTypeCli::Normalized) { MbtTypeCli::Flat => MbtType::Flat, MbtTypeCli::FlatWithHash => MbtType::FlatWithHash, MbtTypeCli::Normalized => MbtType::Normalized { hash_view: true }, @@ -383,12 +387,19 @@ async fn init_schema( let mut tj = merge_tilejson(sources, String::new()); tj.other.insert( "format".to_string(), - serde_json::Value::String(tile_info.format.to_string()), + serde_json::Value::String(tile_info.format.metadata_format_value().to_string()), ); tj.other.insert( "generator".to_string(), serde_json::Value::String(format!("martin-cp v{VERSION}")), ); + let zooms = get_zooms(args); + if let Some(min_zoom) = zooms.iter().min() { + tj.minzoom = Some(*min_zoom); + } + if let Some(max_zoom) = zooms.iter().max() { + tj.maxzoom = Some(*max_zoom); + } mbt.insert_metadata(&mut *conn, &tj).await?; mbt_type } else { diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index 5f6bf6c78..8c6f44aab 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.8.4" +version = "0.8.5" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 4c7460166..113c573f1 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -63,6 +63,12 @@ enum Commands { /// Diff file diff_file: PathBuf, }, + /// Update metadata to match the content of the file + #[command(name = "meta-update", alias = "update-meta")] + UpdateMetadata { + /// MBTiles file to validate + file: PathBuf, + }, /// Validate tile data if hash of tile data exists in file #[command(name = "validate", alias = "check", alias = "verify")] Validate { @@ -173,6 +179,10 @@ async fn main_int() -> anyhow::Result<()> { } => { apply_patch(src_file, diff_file).await?; } + Commands::UpdateMetadata { file } => { + let mbt = Mbtiles::new(file.as_path())?; + mbt.update_metadata().await?; + } Commands::Validate { file, integrity_check, diff --git a/mbtiles/src/lib.rs b/mbtiles/src/lib.rs index 30be6032f..f6ce20f19 100644 --- a/mbtiles/src/lib.rs +++ b/mbtiles/src/lib.rs @@ -26,6 +26,8 @@ pub use queries::*; mod summary; +mod update; + mod validation; pub use validation::{ calc_agg_tiles_hash, AggHashType, IntegrityCheckType, MbtType, AGG_TILES_HASH, diff --git a/mbtiles/src/update.rs b/mbtiles/src/update.rs new file mode 100644 index 000000000..c6af501b9 --- /dev/null +++ b/mbtiles/src/update.rs @@ -0,0 +1,24 @@ +use log::info; + +use crate::errors::MbtResult; +use crate::Mbtiles; + +impl Mbtiles { + pub async fn update_metadata(&self) -> MbtResult<()> { + let mut conn = self.open().await?; + let info = self.summary(&mut conn).await?; + + if let Some(min_zoom) = info.min_zoom { + info!("Updating minzoom to {min_zoom}"); + self.set_metadata_value(&mut conn, "minzoom", &min_zoom) + .await?; + } + if let Some(max_zoom) = info.max_zoom { + info!("Updating maxzoom to {max_zoom}"); + self.set_metadata_value(&mut conn, "maxzoom", &max_zoom) + .await?; + } + + Ok(()) + } +} diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 3166fa16a..0e33b9600 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -246,6 +246,16 @@ fn databases() -> Databases { }) } +#[actix_rt::test] +async fn update() -> MbtResult<()> { + let (mbt, mut cn) = new_file_no_hash!(databases, Flat, METADATA_V1, TILES_V1, "update"); + mbt.update_metadata().await?; + let dmp = dump(&mut cn).await?; + assert_snapshot!(&dmp, "update"); + + Ok(()) +} + #[rstest] #[trace] #[actix_rt::test] diff --git a/mbtiles/tests/snapshots/copy__update@update.snap b/mbtiles/tests/snapshots/copy__update@update.snap new file mode 100644 index 000000000..6f96ba28c --- /dev/null +++ b/mbtiles/tests/snapshots/copy__update@update.snap @@ -0,0 +1,51 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "maxzoom", "6" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', + '( "minzoom", "3" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(root) )', + '( 5, 0, 0, blob(same) )', + '( 5, 0, 1, blob() )', + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 1, 3, blob(non-empty) )', + '( 5, 2, 2, blob(remove) )', + '( 5, 2, 3, blob() )', + '( 6, 0, 3, blob(same) )', + '( 6, 0, 5, blob(1-keep-1-rm) )', + '( 6, 1, 4, blob(edit-v1) )', + '( 6, 2, 6, blob(1-keep-1-rm) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/tests/expected/martin-cp/flat-with-hash_metadata.txt b/tests/expected/martin-cp/flat-with-hash_metadata.txt index d2a1761dd..2a43b30de 100644 --- a/tests/expected/martin-cp/flat-with-hash_metadata.txt +++ b/tests/expected/martin-cp/flat-with-hash_metadata.txt @@ -7,8 +7,10 @@ tilejson: tilejson: 3.0.0 tiles: [] description: public.function_zxy_query_test + maxzoom: 6 + minzoom: 0 name: function_zxy_query_test - format: mvt + format: pbf generator: martin-cp v0.0.0 agg_tiles_hash: 9B931A386D6075D1DA55323BD4DBEDAE diff --git a/tests/expected/martin-cp/flat_metadata.txt b/tests/expected/martin-cp/flat_metadata.txt index 0740ff58d..46dc17d93 100644 --- a/tests/expected/martin-cp/flat_metadata.txt +++ b/tests/expected/martin-cp/flat_metadata.txt @@ -15,9 +15,11 @@ tilejson: - -1.0 - 142.84131509869133 - 45.0 + maxzoom: 6 + minzoom: 0 name: table_source foo: '{"bar":"foo"}' - format: mvt + format: pbf generator: martin-cp v0.0.0 agg_tiles_hash: EF19FCBCE73ADE1C85E856E6BBA9B4C7 diff --git a/tests/expected/martin-cp/normalized_metadata.txt b/tests/expected/martin-cp/normalized_metadata.txt index 370a0bd6a..955f59d88 100644 --- a/tests/expected/martin-cp/normalized_metadata.txt +++ b/tests/expected/martin-cp/normalized_metadata.txt @@ -23,7 +23,7 @@ tilejson: - maxzoom: 1 + maxzoom: 6 minzoom: 0 name: normalized template: |- From 9a9091f1c1651e2061a52f1169c3e1b782c6a0e2 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 18 Dec 2023 22:42:23 -0500 Subject: [PATCH 2/2] simplify update query --- ...bba64268170ab6e5381abfe07df24d8133229.json | 26 +++++++++++++++++++ mbtiles/src/update.rs | 11 +++++++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 mbtiles/.sqlx/query-47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229.json diff --git a/mbtiles/.sqlx/query-47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229.json b/mbtiles/.sqlx/query-47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229.json new file mode 100644 index 000000000..a1be09ebe --- /dev/null +++ b/mbtiles/.sqlx/query-47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT min(zoom_level) AS min_zoom,\n max(zoom_level) AS max_zoom\n FROM tiles", + "describe": { + "columns": [ + { + "name": "min_zoom", + "ordinal": 0, + "type_info": "Int" + }, + { + "name": "max_zoom", + "ordinal": 1, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + true, + true + ] + }, + "hash": "47bdc12fe7b34fb2e4e1fc3b937bba64268170ab6e5381abfe07df24d8133229" +} diff --git a/mbtiles/src/update.rs b/mbtiles/src/update.rs index c6af501b9..5199f9ad9 100644 --- a/mbtiles/src/update.rs +++ b/mbtiles/src/update.rs @@ -1,4 +1,5 @@ use log::info; +use sqlx::query; use crate::errors::MbtResult; use crate::Mbtiles; @@ -6,7 +7,15 @@ use crate::Mbtiles; impl Mbtiles { pub async fn update_metadata(&self) -> MbtResult<()> { let mut conn = self.open().await?; - let info = self.summary(&mut conn).await?; + + let info = query!( + " + SELECT min(zoom_level) AS min_zoom, + max(zoom_level) AS max_zoom + FROM tiles" + ) + .fetch_one(&mut conn) + .await?; if let Some(min_zoom) = info.min_zoom { info!("Updating minzoom to {min_zoom}");