* Document changes in README

* Add -f to force geometry updates * Print sizes of files
instedd · May 31, 2024 · 2a7bfac · 2a7bfac
1 parent 1f271c5
commit 2a7bfac
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 19 deletions.
diff --git a/scripts/population/README.md b/scripts/population/README.md
@@ -15,6 +15,19 @@ $ docker-compose run --rm app bash
 /app$ scripts/population/load-regions URY Uruguay
 ```
 
+### Geometry simplification
+
+The script simplifies the regions using mapshaper by a distance between points.
+We can adjust this parameter with the environment `INTERVAL_DISTANCE`.
+
+For example to simplify with 120 meters distance
+
+```sh
+$ INTERVAL_DISTANCE=120 ./load-regions -f URY Uruguay
+```
+
+Here `-f` is used to force updating an existing region.
+
 ## Clip friction layer for new regions
 
 The friction raster is clipped to the country-level regions for quicker access
@@ -46,4 +59,3 @@ INSERT INTO source_set (name, type, unit, raster_file) VALUES ('Uruguay PPP v2b
 
 The important bit of information is the last field. That should be a relative
 path from the `DATA_PATH` directory to the downloaded raster file.
-
diff --git a/scripts/population/load-regions b/scripts/population/load-regions
@@ -4,12 +4,35 @@ set -euo pipefail
 export PGPASSWORD=$POSTGRES_PASSWORD;
 
 if [ $# -lt 2 ]; then
-    echo "Usage $0 <ISO> <Country Name>"
+    echo "Usage $0 [-f] <ISO> <Country Name>"
+    echo "The option -f forces updating an existing country geometry"
     exit 1
 fi
 
-COUNTRY_CODE=$1
-COUNTRY_NAME=$2
+FORCE=
+COUNTRY_CODE=
+COUNTRY_NAME=
+
+while [ $# -gt 0 ]; do
+    case $1 in
+        -f|--force)
+            FORCE=1
+            ;;
+        *)
+            if [ ! -z "$COUNTRY_NAME" ]; then
+                echo Unknown arguments specified
+                exit 1;
+            elif [ ! -z "$COUNTRY_CODE" ]; then
+                COUNTRY_NAME=$1
+            else
+                COUNTRY_CODE=$1
+            fi
+            ;;
+    esac
+    shift
+done
+
+
 #LEVELS=${2:-2,4}
 LEVELS=(0,1)
 echo "   -> Importing $COUNTRY_NAME at levels $LEVELS"
@@ -18,7 +41,7 @@ echo "   -> Importing $COUNTRY_NAME at levels $LEVELS"
 REGIONS=$(psql -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT -t -A -c "SELECT id, country FROM regions WHERE country = '$COUNTRY_NAME';")
 REGIONS_LEN=$(echo $REGIONS | wc -w | tr -d '[[:space:]]')
 
-if [ ${REGIONS_LEN} -gt 0 ]; then
+if [ ${REGIONS_LEN} -gt 0 -a -z $FORCE ]; then
   echo "   -> Regions for country $COUNTRY_NAME already imported."
   exit 0
 fi
@@ -37,9 +60,12 @@ fi
 # From mapshaper wiki https://github.com/mbloch/mapshaper/wiki/Command-Reference#-simplify
 #
 # interval= Specify simplification amount in units of distance.
-#           Uses meters when simplifying unprojected datasets in 3D space (see planar option below),
+#           Uses meters when simplifying unprojected datasets in 3D space,
 #           otherwise uses the same units as the source data.
-INTERVAL_DISTANCE=${INTERVAL_DISTANCE-1000}
+#           In our case since our GeoJSON are not projected (ie. use WGS84 coordinates)
+#           then this means the resolution for the simplified shape will be in meters.
+
+INTERVAL_DISTANCE=${INTERVAL_DISTANCE-100}
 
 IFS=',';
 for i in $LEVELS; do
@@ -52,23 +78,49 @@ for i in $LEVELS; do
     OFILE=${DATA_PATH}/${COUNTRY_CODE}/${COUNTRY_CODE}_adm${i}_${INTERVAL_DISTANCE}.geojson
 
     mapshaper $FILE -simplify interval=$INTERVAL_DISTANCE -o $OFILE
+
+    SIZE=$(stat --printf="%s" $FILE)
+    OSIZE=$(stat --printf="%s" $OFILE)
+
+    echo "   -> Finished file size went from $SIZE to $OSIZE"
   fi
 
-  psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT
+  if [ "$FORCE" = "1" ]; then
+    psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT
+
+      WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc)
+      UPDATE "regions" SET the_geom=f.the_geom
+      FROM (
+        SELECT
+          '${COUNTRY_NAME}' as country,
+          feat#>>'{properties,name}' AS name,
+          ${i} as admin_level,
+          ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom
+        FROM (
+          SELECT json_array_elements(fc->'features') AS feat
+          FROM data
+        ) AS dummy
+      ) AS f
+      WHERE regions.country=f.country AND regions.name=f.name AND regions.admin_level=f.admin_level;
 
-    WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc)
-    INSERT INTO "regions" (country, name, admin_level, the_geom)
-    SELECT
-      '${COUNTRY_NAME}',
-      feat#>>'{properties,name}' AS name,
-      ${i},
-      ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom
-    FROM (
-      SELECT json_array_elements(fc->'features') AS feat
-      FROM data
-    ) AS f;
+SQL_SCRIPT
+  else
+    psql -q -d $POSTGRES_DB -U $POSTGRES_USER -h $POSTGRES_HOST -p $POSTGRES_PORT << SQL_SCRIPT
+
+      WITH data AS (SELECT \$$`cat $OFILE`\$$::json AS fc)
+      INSERT INTO "regions" (country, name, admin_level, the_geom)
+      SELECT
+        '${COUNTRY_NAME}',
+        feat#>>'{properties,name}' AS name,
+        ${i},
+        ST_SetSRID(ST_CollectionExtract(ST_Multi(ST_GeomFromGeoJSON(feat->>'geometry')), 3), 4326) as the_geom
+      FROM (
+        SELECT json_array_elements(fc->'features') AS feat
+        FROM data
+      ) AS f;
 
 SQL_SCRIPT
+  fi
   # TODO: print errors
   echo " done!"
 done;