Add newer version of municipalities (#197)

arpav_ppcv/bootstrapper/cliapp.py

@@ -36,30 +36,96 @@
 
 
 @app.command("municipalities")
-def bootstrap_municipalities(ctx: typer.Context) -> None:
-    """Bootstrap Italian municipalities."""
-    data_directory = Path(__file__).parents[2] / "data"
-    municipalities_dataset = data_directory / "limits_IT_municipalities.geojson"
+def bootstrap_municipalities(
+    ctx: typer.Context, municipalities_dataset: Path, force: bool = False
+) -> None:
+    """Bootstrap Italian municipalities"""
+    # data_directory = Path(__file__).parents[2] / "data"
+    # municipalities_dataset = data_directory / "limits_IT_municipalities.geojson"
     to_create = []
-    with municipalities_dataset.open() as fh:
-        municipalities_geojson = json.load(fh)
-        for idx, feature in enumerate(municipalities_geojson["features"]):
-            print(
-                f"parsing feature ({idx+1}/{len(municipalities_geojson['features'])})..."
-            )
-            props = feature["properties"]
-            mun_create = municipalities.MunicipalityCreate(
-                geom=geojson_pydantic.MultiPolygon(
-                    type="MultiPolygon", coordinates=feature["geometry"]["coordinates"]
-                ),
-                name=props["name"],
-                province_name=props["prov_name"],
-                region_name=props["reg_name"],
-            )
-            to_create.append(mun_create)
-    print(f"About to save {len(to_create)} municipalities...")
+
+    should_bootstrap = False
+    with sqlmodel.Session(ctx.obj["engine"]) as session:
+        _, num_existing_municipalities = database.list_municipalities(
+            session, include_total=True
+        )
+        if num_existing_municipalities == 0:
+            should_bootstrap = True
+        else:
+            if force:
+                should_bootstrap = True
+            else:
+                print(
+                    "Municipalities have already been bootstrapped. Supply the "
+                    "`--force` option to discard existing records and re-boostrap "
+                    "them again"
+                )
+        if should_bootstrap:
+            has_centroid_info = False
+            with municipalities_dataset.open() as fh:
+                municipalities_geojson = json.load(fh)
+                for idx, feature in enumerate(municipalities_geojson["features"]):
+                    print(
+                        f"parsing feature ({idx + 1}/{len(municipalities_geojson['features'])})..."
+                    )
+                    props = feature["properties"]
+                    if idx == 0:
+                        has_centroid_info = props.get("xcoord") is not None
+                    mun_create = municipalities.MunicipalityCreate(
+                        geom=geojson_pydantic.MultiPolygon(
+                            type="MultiPolygon",
+                            coordinates=feature["geometry"]["coordinates"],
+                        ),
+                        name=props["name"],
+                        province_name=props["province_name"],
+                        region_name=props["region_name"],
+                        centroid_epsg_4326_lon=props.get("xcoord"),
+                        centroid_epsg_4326_lat=props.get("ycoord"),
+                    )
+                    to_create.append(mun_create)
+            if len(to_create) > 0:
+                if num_existing_municipalities > 0:
+                    print("About to delete pre-existing municipalities...")
+                    database.delete_all_municipalities(session)
+                print(f"About to save {len(to_create)} municipalities...")
+                database.create_many_municipalities(session, to_create)
+                if has_centroid_info:
+                    print("About to (re)create municipality centroids DB view...")
+                    ctx.invoke(bootstrap_municipality_centroids, ctx)
+            else:
+                print("There are no municipalities to create, skipping...")
+    print("Done!")
+
+
+@app.command("municipality-centroids")
+def bootstrap_municipality_centroids(
+    ctx: typer.Context,
+):
+    """Refresh the municipality centroids' DB view."""
+    view_name = "public.municipality_centroids"
+    index_name = "idx_municipality_centroids"
+    drop_view_statement = sqlmodel.text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")
+    create_view_statement = sqlmodel.text(
+        f"CREATE MATERIALIZED VIEW {view_name} "
+        f"AS SELECT "
+        f"id, "
+        f"ST_Point(centroid_epsg_4326_lon, centroid_epsg_4326_lat, 4326) AS geom, "
+        f"name, "
+        f"province_name, "
+        f"region_name "
+        f"FROM municipality "
+        f"WITH DATA"
+    )
+    create_index_statement = sqlmodel.text(
+        f"CREATE INDEX {index_name} ON {view_name} USING gist (geom)"
+    )
+    drop_index_statement = sqlmodel.text(f"DROP INDEX IF EXISTS {index_name}")
     with sqlmodel.Session(ctx.obj["engine"]) as session:
-        database.create_many_municipalities(session, to_create)
+        session.execute(drop_view_statement)
+        session.execute(drop_index_statement)
+        session.execute(create_view_statement)
+        session.execute(create_index_statement)
+        session.commit()
     print("Done!")
 
 

arpav_ppcv/database.py

@@ -1207,7 +1207,7 @@ def list_municipalities(
     name_filter: Optional[str] = None,
     province_name_filter: Optional[str] = None,
     region_name_filter: Optional[str] = None,
-) -> Optional[municipalities.Municipality]:
+) -> tuple[Sequence[municipalities.Municipality], Optional[int]]:
     """List existing municipalities.
 
     Both ``polygon_intersection_filter`` and ``point_filter`` parameters are expected
@@ -1249,6 +1249,14 @@ def list_municipalities(
     return items, num_items
 
 
+def collect_all_municipalities(
+    session: sqlmodel.Session,
+) -> Sequence[municipalities.Municipality]:
+    _, num_total = list_municipalities(session, limit=1, include_total=True)
+    result, _ = list_municipalities(session, limit=num_total, include_total=False)
+    return result
+
+
 def create_many_municipalities(
     session: sqlmodel.Session,
     municipalities_to_create: Sequence[municipalities.MunicipalityCreate],
@@ -1274,6 +1282,13 @@ def create_many_municipalities(
         return db_records
 
 
+def delete_all_municipalities(session: sqlmodel.Session) -> None:
+    """Delete all municipalities."""
+    for db_municipality in collect_all_municipalities(session):
+        session.delete(db_municipality)
+    session.commit()
+
+
 def list_coverage_identifiers(
     session: sqlmodel.Session,
     *,

arpav_ppcv/migrations/versions/2c0e4144c2ec_added_municipality_centroid_coords.py

@@ -0,0 +1,33 @@
+"""added-municipality-centroid-coords
+
+Revision ID: 2c0e4144c2ec
+Revises: d445c73f5aef
+Create Date: 2024-08-20 18:09:27.462147
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = '2c0e4144c2ec'
+down_revision: Union[str, None] = 'd445c73f5aef'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('municipality', sa.Column('centroid_epsg_4326_lon', sa.Float(), nullable=True))
+    op.add_column('municipality', sa.Column('centroid_epsg_4326_lat', sa.Float(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('municipality', 'centroid_epsg_4326_lat')
+    op.drop_column('municipality', 'centroid_epsg_4326_lon')
+    # ### end Alembic commands ###

arpav_ppcv/schemas/municipalities.py

@@ -25,10 +25,14 @@ class Municipality(sqlmodel.SQLModel, table=True):
     name: str
     province_name: str
     region_name: str
+    centroid_epsg_4326_lon: float | None = None
+    centroid_epsg_4326_lat: float | None = None
 
 
 class MunicipalityCreate(sqlmodel.SQLModel):
     geom: geojson_pydantic.MultiPolygon
     name: str
     province_name: str
     region_name: str
+    centroid_epsg_4326_lon: float | None = None
+    centroid_epsg_4326_lat: float | None = None

arpav_ppcv/webapp/api_v2/routers/municipalities.py

@@ -55,7 +55,7 @@ def list_municipalities(
         region_name_filter=region,
         **geom_filter_kwarg,
     )
-    _, unfiltered_total = db.list_stations(
+    _, unfiltered_total = db.list_municipalities(
         db_session, limit=1, offset=0, include_total=True
     )
     return municipalities_geojson.MunicipalityFeatureCollection.from_items(

data/README.md

@@ -0,0 +1,53 @@
+# ARPAV-PPCV auxiliary datasets
+
+
+### Italian municipalities
+
+The `municipalities-istat-2021.geojson` file holds a dataset with italian municipalities that are part of the
+relevant regions for the ARPAV-PPCV system, namely:
+
+- Friuli Venezia Giulia
+- Trentino-Alto Adige
+- Veneto
+
+The ARPAV-PPCV system expects each municipality feature to have the following properties:
+
+- `fid`
+- `name`
+- `province_name`
+- `region_name`
+- `xcoord` - longitude of the centroid, in EPSG:4326
+- `ycoord` - latitude of the centroid, in EPSG:4326
+
+The system also expects features to have a geometry type of `MultiPolygon`
+
+
+This dataset has been prepared from the official ISTAT datasets, which are made available publicly at:
+
+    https://www.istat.it/notizia/confini-delle-unita-amministrative-a-fini-statistici-al-1-gennaio-2018-2/
+
+The dataset currently in use is the one for the year 2021. This was chosen as this was the latest bundle which
+included the relevant centroid information for each municipality in the companion .xlsx file.
+
+The original downloaded dataset consisted of a zingle zip file which decompressed into multiple files, for which the
+following were relevant:
+
+- ProvCM2021/ProvCM2021.sh* - geospatial boundaries of all italian municipalities
+- ElencoUnitaAmministrative2021.xlsx - relevant alphanumerical information about municipalities, including the
+  coordinates of the points which should be regarded as the centroid
+
+The original dataset was pre-processed into the `municipalities-istat-2021.geojson` present in this directory.
+Pre-processing consisted mainly in the following operations:
+
+- Discard all municipalities which do not belong to the any of the target regions
+- Include the name of the province and the name of the region for each municipality
+- Extract the x and y coordinates from the .xlsx which represent municipality centroids and join them with the
+  respective municipality
+- Reproject the geometries and also the x and and y coordinates of centroids to `EPSG:4326`
+- Save the result as a GeoJSON file
+
+This dataset is ingested into the ARPAV-PPCV system database by running the command:
+
+```shell
+arpav-ppcv bootstrap municipalities data/municipalities-istat-2021.geojson
+```

docker/martin/config.yaml

@@ -31,3 +31,18 @@ postgres:
         name: varchar
         province_name: varchar
         region_name: varchar
+        centroid_epsg_4326_lon: float
+        centroid_epsg_4326_lat: float
+
+    municipality-centroids:
+      schema: public
+      table: municipality_centroids
+      srid: 4326
+      geometry_column: geom
+      id_column: ~
+      geometry_type: POINT
+      properties:
+        id: uuid
+        name: varchar
+        province_name: varchar
+        region_name: varchar