wip

mbtiles/src/bin/mbtiles.rs

@@ -56,9 +56,9 @@ enum Commands {
     /// Compare two files A and B, and generate a new diff file. If the diff file is applied to A, it will produce B.
     #[command(name = "diff")]
     Diff {
-        file_a: PathBuf,
-        file_b: PathBuf,
-        diff_file: PathBuf,
+        file: PathBuf,
+        diff_output: PathBuf,
+        diff_with: PathBuf,
     },
     /// Copy tiles from one mbtiles file to another.
     #[command(name = "copy", alias = "cp")]
@@ -221,9 +221,9 @@ async fn main_int() -> anyhow::Result<()> {
             println!("{}", mbt.summary(&mut conn).await?);
         }
         Commands::Diff {
-            file_a,
-            file_b,
-            diff_file,
+            file: file_a,
+            diff_output: file_b,
+            diff_with: diff_file,
         } => diff(file_a, file_b, diff_file).await?,
     }
 

mbtiles/src/diff.rs

@@ -1,95 +1,200 @@
+use log::debug;
+use sqlite_hashes::rusqlite::Connection;
 use std::path::PathBuf;
 
-use sqlx::SqliteConnection;
+use crate::MbtType::{Flat, FlatWithHash, Normalized};
+use crate::{is_empty_database, CopyDuplicateMode, MbtError, MbtResult, MbtType, Mbtiles};
 
-use crate::{MbtError, MbtResult, MbtType, Mbtiles};
+pub const AGG_TILES_HASH: &str = "agg_tiles_hash";
 
-pub async fn diff(file_a: PathBuf, file_b: PathBuf, file_diff: PathBuf) -> MbtResult<()> {
-    if file_a == file_b {
-        return Err(MbtError::SameSourceAndDestination(file_a.clone()));
+pub const AGG_TILES_HASH_IN_DIFF: &str = "agg_tiles_hash_after_apply";
+pub async fn diff(src: PathBuf, dst: PathBuf, diff_with: PathBuf) -> MbtResult<()> {
+    if src == dst {
+        return Err(MbtError::SameSourceAndDestination(src.clone()));
     }
 
-    if file_diff.exists() {
-        return Err(MbtError::DiffFileExists(file_diff));
+    if dst.exists() {
+        return Err(MbtError::DestinationFileExists(dst.clone()));
     }
 
-    let src_mbt = Mbtiles::new(file_a)?;
-    let dst_mbt = Mbtiles::new(file_b)?;
-    let diff_mbt = Mbtiles::new(file_diff)?;
-
-    let mut diff_conn = diff_mbt.open_or_new().await?;
-
+    let src_mbt = Mbtiles::new(src)?;
     let src_type = src_mbt.open_and_detect_type().await?;
-    let dst_type = dst_mbt.open_and_detect_type().await?;
 
-    let mut conn = dst_mbt.open_or_new().await?;
-    src_mbt.attach_to(&mut conn, "sourceDb").await?;
-    diff_mbt.attach_to(&mut conn, "diffDb").await?;
+    let dif_mbt = Mbtiles::new(diff_with)?;
+    let dif_type = dif_mbt.open_and_detect_type().await?;
 
-    init_mbtiles(&mut diff_conn, src_type).await?;
+    let dst_mbt = Mbtiles::new(dst.clone())?;
+    let mut conn = dst_mbt.open_or_new().await?;
+    if !is_empty_database(&mut conn).await? {
+        return Err(MbtError::DestinationFileExists(dst.clone()));
+    }
+    let dst_type = src_type.clone();
 
-    write_tiles_diff(&mut conn, src_type, dst_type, "sourceDb", "diffDb").await?;
-    write_metadata_diff(&mut conn, src_type, dst_type, "sourceDb", "diffDb").await?;
+    src_mbt.attach_to(&mut conn, "sourceDb").await?;
+    dst_mbt.attach_to(&mut conn, "diffDb").await?;
+
+    {
+        let mut handle_lock = conn.lock_handle().await?;
+        let handle = handle_lock.as_raw_handle().as_ptr();
+
+        // SAFETY: this is safe as long as handle_lock is valid. We will drop the lock.
+        let rusqlite_conn = unsafe { Connection::from_handle(handle) }?;
+        write_tiles_diff(
+            &rusqlite_conn,
+            dif_type,
+            src_type,
+            dst_type,
+            CopyDuplicateMode::Override,
+        )
+        .await?;
+        write_meta_diff(&rusqlite_conn, dif_type, CopyDuplicateMode::Override).await?;
+    }
 
     todo!()
 }
-
-async fn write_metadata_diff(
-    conn: &mut SqliteConnection,
-    src_type: MbtType,
-    dst_type: MbtType,
-    src_attach_name: &str,
-    dst_attach_name: &str,
-) -> MbtResult<()> {
-    todo!()
+async fn write_meta_diff(
+    rusqlite_conn: &Connection,
+    dif_type: MbtType,
+    on_duplicate: CopyDuplicateMode,
+) -> Result<(), MbtError> {
+    let on_dupl = "OR REPLACE";
+    let sql;
+    sql = format!(
+        "
+    INSERT {on_dupl} INTO metadata (name, value)
+        SELECT IIF(name = '{AGG_TILES_HASH}','{AGG_TILES_HASH_IN_DIFF}', name) as name
+             , value
+        FROM (
+            SELECT COALESCE(difMD.name, srcMD.name) as name
+                 , difMD.value as value
+            FROM sourceDb.metadata AS srcMD FULL JOIN diffDb.metadata AS difMD
+                 ON srcMD.name = difMD.name
+            WHERE srcMD.value != difMD.value OR srcMD.value ISNULL OR difMD.value ISNULL
+        ) joinedMD
+        WHERE name != '{AGG_TILES_HASH_IN_DIFF}'"
+    );
+    rusqlite_conn.execute(&sql, [])?;
+    Ok(())
 }
-
 async fn write_tiles_diff(
-    conn: &mut SqliteConnection,
+    rusqlite_conn: &Connection,
+    diff_type: MbtType,
     src_type: MbtType,
     dst_type: MbtType,
-    src_attach_name: &str,
-    dst_attach_name: &str,
-) -> MbtResult<()> {
-    let tile_diff_sql = match (src_type, dst_type) {
-        (MbtType::Flat, MbtType::Flat) => include_str!("scripts/diff/flat_to_others.sql"),
-        (MbtType::Flat, MbtType::FlatWithHash) =>include_str!("scripts/diff/flat_to_others.sql"),
-        (MbtType::Flat, MbtType::Normalized { hash_view }) =>include_str!("scripts/diff/flat_to_others.sql"),
-        (MbtType::FlatWithHash, MbtType::Flat) => todo!(),
-        (MbtType::FlatWithHash, MbtType::FlatWithHash) => todo!(),
-        (MbtType::FlatWithHash, MbtType::Normalized { hash_view }) => todo!(),
-        (MbtType::Normalized { hash_view }, MbtType::Flat) => todo!(),
-        (MbtType::Normalized { hash_view }, MbtType::FlatWithHash) => todo!(),
-        (
-            MbtType::Normalized {
-                hash_view: hash_view1,
-            },
-            MbtType::Normalized {
-                hash_view: has_view2,
-            },
-        ) => todo!(),
+    on_duplicate: CopyDuplicateMode,
+) -> Result<(), MbtError> {
+    let on_dupl = on_duplicate.to_sql();
+
+    let select_from = get_select_from_with_diff(diff_type.clone(), dst_type);
+    let sql_cond = get_on_duplicate_sql_cond(on_duplicate, dst_type);
+    let sql = match dst_type {
+        Flat => {
+            format!(
+                "
+    INSERT {on_dupl} INTO tiles
+           (zoom_level, tile_column, tile_row, tile_data)
+    {select_from}  {sql_cond}"
+            )
+        }
+        FlatWithHash => {
+            format!(
+                "
+    INSERT {on_dupl} INTO tiles_with_hash
+           (zoom_level, tile_column, tile_row, tile_data, tile_hash)
+    {select_from}  {sql_cond}"
+            )
+        }
+        Normalized { .. } => {
+            let sql = format!(
+                "
+    INSERT OR IGNORE INTO images
+           (tile_id, tile_data)
+    SELECT tile_hash as tile_id, tile_data
+    FROM ({select_from} )"
+            );
+            debug!("Copying to {dst_type} with {sql}");
+            rusqlite_conn.execute(&sql, [])?;
+
+            format!(
+                "
+    INSERT {on_dupl} INTO map
+           (zoom_level, tile_column, tile_row, tile_id)
+    SELECT zoom_level, tile_column, tile_row, tile_hash as tile_id
+    FROM ({select_from} {sql_cond})"
+            )
+        }
     };
-    sqlx::query(tile_diff_sql).execute(conn).await?;
+
+    rusqlite_conn.execute(&sql, [])?;
     Ok(())
 }
+/// Returns WHERE condition SQL depending on the override and destination type
+fn get_on_duplicate_sql_cond(on_duplicate: CopyDuplicateMode, dst_type: MbtType) -> String {
+    match on_duplicate {
+        CopyDuplicateMode::Ignore | CopyDuplicateMode::Override => String::new(),
+        CopyDuplicateMode::Abort => {
+            let (main_table, tile_identifier) = match dst_type {
+                Flat => ("tiles", "tile_data"),
+                FlatWithHash => ("tiles_with_hash", "tile_data"),
+                Normalized { .. } => ("map", "tile_id"),
+            };
+
+            format!(
+                "AND NOT EXISTS (
+                             SELECT 1
+                             FROM {main_table}
+                             WHERE
+                                 {main_table}.zoom_level = sourceDb.{main_table}.zoom_level
+                                 AND {main_table}.tile_column = sourceDb.{main_table}.tile_column
+                                 AND {main_table}.tile_row = sourceDb.{main_table}.tile_row
+                                 AND {main_table}.{tile_identifier} != sourceDb.{main_table}.{tile_identifier}
+                         )"
+            )
+        }
+    }
+}
 
-async fn init_mbtiles(conn: &mut SqliteConnection, mbt_type: MbtType) -> MbtResult<()> {
-    let sql = match mbt_type {
-        MbtType::Flat => include_str!("scripts/flat.sql"),
-        MbtType::FlatWithHash => include_str!("scripts/flat-with-hash.sql"),
-        MbtType::Normalized { hash_view } => match hash_view {
-            true => include_str!("scripts/normalized_with_view.sql"),
-            false => include_str!("scripts/normalized.sql"),
-        },
-    };
+fn get_select_from_with_diff(dif_type: MbtType, dst_type: MbtType) -> String {
+    let tile_hash_expr;
+    let diff_tiles;
+    if dst_type == Flat {
+        tile_hash_expr = "";
+        diff_tiles = "diffDb.tiles";
+    } else {
+        tile_hash_expr = match dif_type {
+            Flat => ", COALESCE(md5_hex(difTiles.tile_data), '') as tile_hash",
+            FlatWithHash | Normalized { .. } => ", COALESCE(difTiles.tile_hash, '') as tile_hash",
+        };
+        diff_tiles = match dif_type {
+            Flat => "diffDb.tiles",
+            FlatWithHash => "diffDb.tiles_with_hash",
+            Normalized { .. } => {
+                "
+        (SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS tile_hash
+        FROM diffDb.map JOIN diffDb.images ON diffDb.map.tile_id = diffDb.images.tile_id)"
+            }
+        };
+    }
 
-    sqlx::query(sql).execute(conn).await?;
-    Ok(())
+    format!(
+        "
+        SELECT COALESCE(srcTiles.zoom_level, difTiles.zoom_level) as zoom_level
+             , COALESCE(srcTiles.tile_column, difTiles.tile_column) as tile_column
+             , COALESCE(srcTiles.tile_row, difTiles.tile_row) as tile_row
+             , difTiles.tile_data as tile_data
+             {tile_hash_expr}
+        FROM sourceDb.tiles AS srcTiles FULL JOIN {diff_tiles} AS difTiles
+             ON srcTiles.zoom_level = difTiles.zoom_level
+               AND srcTiles.tile_column = difTiles.tile_column
+               AND srcTiles.tile_row = difTiles.tile_row
+        WHERE (srcTiles.tile_data != difTiles.tile_data
+               OR srcTiles.tile_data ISNULL
+               OR difTiles.tile_data ISNULL)"
+    )
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::diff::init_mbtiles;
     use sqlx::Connection;
     use std::path::PathBuf;
 
@@ -98,8 +203,8 @@ mod tests {
     #[actix_rt::test]
     async fn diff_flat_to_flat() -> MbtResult<()> {
         let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles");
-        let dst = PathBuf::from("../tests/fixtures/mbtiles/json.mbtiles");
-        let diff = PathBuf::from("diff_flat_to_flat.mbtiles");
+        let diff = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles");
+        let dst = PathBuf::from("diff_flat_to_flat.mbtiles");
 
         super::diff(src, dst, diff).await
     }