Update to OSM-prebuilt v0.6 (#1431)

* Implemented line merge over virtual bus functionality.

* Implemented: Aggregating identical lines, bus merging to stations, creating voltage-specific bus endings in stations.

* Implemented: Mapping of lines to buses and extending lines to buses.

* Finished implementing converters and links.

* Finished implementation of entirely new build_osm_network.py script.

* Updated configtables.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Temporarily disabled tqdm in retrieve. Line splitting disabled as well.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Implemented relations. Running, converging workflow.

* Added updated simplify_network.py

* Cleaned up code.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Updated doc

* Reactivated line splitting.

* Updated build_osm_network.

* Cleaned up code.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Added 400 kV voltage level to config.default

* Updated Zenodo link

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Cleaned up code based on PR comments

* Added docstrings.

* Updated prepare_osm_network to include interactive map. Added electric parameters for usage outside PyPSA/PyPSA-Eur

* Updated config.default. Removed dummy configs.

* Updated retriever to include map.html

* Add custom busmaps

* Added GB busmaps, 37 is london clustered. 314 is london clustered for europe

* Removed data files, updated docs.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Fabian Neumann <fabian.neumann@outlook.de>

config/config.default.yaml

@@ -86,7 +86,7 @@ co2_budget:
 electricity:
   voltages: [220., 300., 330., 380., 400., 500., 750.]
   base_network: osm-prebuilt
-  osm-prebuilt-version: 0.5
+  osm-prebuilt-version: 0.6
   gaslimit_enable: false
   gaslimit: false
   co2limit_enable: false

doc/configtables/electricity.csv

@@ -1,7 +1,7 @@
 ,Unit,Values,Description
 voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider
 base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer."
-osm-prebuilt-version, --, "float, any value in range 0.1-0.5", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
+osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
 gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
 co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks..

doc/data-base-network.rst

@@ -15,11 +15,11 @@ The map might take a moment to load. To view it in full screen, click `here <bas
 
 ``data/osm-prebuilt``
 
-- **Source:** OpenStreetMap; Xiong, B., Neumann, F., & Brown, T. (2024).
+- **Source:** OpenStreetMap; Xiong, B., Fioriti, D., Neumann, F., Riepin I. & Brown, T. (2024).
   Prebuilt Electricity Network for PyPSA-Eur based on OpenStreetMap Data (0.5)
-  [Data set]. Zenodo. https://doi.org/10.5281/zenodo.13981528
-- **Link:** https://zenodo.org/records/13981528
-- **License:** ODbL (`reference <https://zenodo.org/records/13981528>`)
+  [Data set]. Zenodo. https://doi.org/10.5281/zenodo.14144752
+- **Link:** https://zenodo.org/records/14144752
+- **License:** ODbL (`reference <https://zenodo.org/records/14144752>`)
 - **Description:** Pre-built data of high-voltage transmission grid in Europe from OpenStreetMap.
 
 This dataset contains a topologically connected representation of the European

doc/release_notes.rst

@@ -103,7 +103,7 @@ Upcoming Release
   - Single transformers for each combination of voltage level per substation. Transformers now have a capacity s_nom based on connected lines
   - Use of OSM relations where available and unambiguous (Overwriting all lines that are members of the respective relation to avoid duplicates)
 
-* Updated osm-prebuilt base network to version 0.5, for changelog, see https://zenodo.org/records/13981528
+* Updated osm-prebuilt base network to version 0.6, for changelog, see https://zenodo.org/records/14144752
 
 * Bugfix: vehicle-to-grid dispatch capacity is now limited by the fraction of vehicles participating in demand-side-management, halving the dispatch capacity under the default demand-side management participation rate of 0.5.
 

rules/build_electricity.smk

@@ -815,6 +815,8 @@ if config["electricity"]["base_network"] == "osm-raw":
             converters_geojson=resources("osm-raw/build/geojson/converters.geojson"),
             transformers_geojson=resources("osm-raw/build/geojson/transformers.geojson"),
             substations_geojson=resources("osm-raw/build/geojson/buses.geojson"),
+            stations_polygon=resources("osm-raw/build/geojson/stations_polygon.geojson"),
+            buses_polygon=resources("osm-raw/build/geojson/buses_polygon.geojson"),
         log:
             logs("build_osm_network.log"),
         benchmark:

rules/development.smk

@@ -5,14 +5,19 @@
 if config["electricity"]["base_network"] == "osm-raw":
 
     rule prepare_osm_network_release:
+        params:
+            line_types=config["lines"]["types"],
         input:
             base_network=resources("networks/base.nc"),
+            stations_polygon=resources("osm-raw/build/geojson/stations_polygon.geojson"),
+            buses_polygon=resources("osm-raw/build/geojson/buses_polygon.geojson"),
         output:
             buses=resources("osm-raw/release/buses.csv"),
             converters=resources("osm-raw/release/converters.csv"),
             lines=resources("osm-raw/release/lines.csv"),
             links=resources("osm-raw/release/links.csv"),
             transformers=resources("osm-raw/release/transformers.csv"),
+            map=resources("osm-raw/release/map.html"),
         log:
             logs("prepare_osm_network_release.log"),
         benchmark:

rules/retrieve.smk

@@ -546,6 +546,7 @@ if config["enable"]["retrieve"] and (
         0.3: "13358976",
         0.4: "13759222",
         0.5: "13981528",
+        0.6: "14144752",
     }
 
     # update rule to use the correct version
@@ -566,12 +567,16 @@ if config["enable"]["retrieve"] and (
             transformers=storage(
                 f"https://zenodo.org/records/{osm_prebuilt_version[config['electricity']['osm-prebuilt-version']]}/files/transformers.csv"
             ),
+            map=storage(
+                f"https://zenodo.org/records/{osm_prebuilt_version[config['electricity']['osm-prebuilt-version']]}/files/map.html"
+            ),
         output:
             buses=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/buses.csv",
             converters=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/converters.csv",
             lines=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/lines.csv",
             links=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/links.csv",
             transformers=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/transformers.csv",
+            map=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/map.html",
         log:
             "logs/retrieve_osm_prebuilt.log",
         threads: 1

scripts/build_osm_network.py

@@ -1429,7 +1429,7 @@ def _finalise_network(all_buses, converters, lines, links, transformers):
     lines_all["voltage"] = lines_all["voltage"] / 1000
     lines_all["length"] = lines_all["length"].round(2)
     lines_all["under_construction"] = False
-    lines_all["tags"] = lines_all["contains_lines"]
+    lines_all["tags"] = lines_all["contains_lines"].apply(lambda x: ";".join(x))
     lines_all["underground"] = lines_all["underground"].replace({True: "t", False: "f"})
     lines_all["under_construction"] = lines_all["under_construction"].replace(
         {True: "t", False: "f"}
@@ -1581,12 +1581,33 @@ def build_network(
     lines["length"] = lines.to_crs(DISTANCE_CRS).length
     links["length"] = links.to_crs(DISTANCE_CRS).length
 
+    # Shapes
+    stations_polygon = stations[["station_id", "geometry"]].copy()
+    all_buses_polygon = buses_polygon.copy()
+    all_buses_polygon["dc"] = False
+    all_buses_polygon = pd.concat(
+        [
+            all_buses_polygon,
+            dc_buses[["bus_id", "polygon", "dc"]].rename(
+                columns={"polygon": "geometry"}
+            ),
+        ]
+    )
+
     ### Saving outputs to PyPSA-compatible format
     buses_final, converters_final, lines_final, links_final, transformers_final = (
         _finalise_network(all_buses, converters, lines, links, transformers)
     )
 
-    return buses_final, converters_final, lines_final, links_final, transformers_final
+    return (
+        buses_final,
+        converters_final,
+        lines_final,
+        links_final,
+        transformers_final,
+        stations_polygon,
+        all_buses_polygon,
+    )
 
 
 if __name__ == "__main__":
@@ -1604,16 +1625,18 @@ def build_network(
     country_shapes = gpd.read_file(snakemake.input["country_shapes"]).set_index("name")
 
     # Build network
-    buses, converters, lines, links, transformers = build_network(
-        snakemake.input,
-        country_shapes,
-        voltages,
-        line_types,
+    buses, converters, lines, links, transformers, stations_polygon, buses_polygon = (
+        build_network(
+            snakemake.input,
+            country_shapes,
+            voltages,
+            line_types,
+        )
     )
 
     # Export to csv for base_network
     buses.to_csv(snakemake.output["substations"], quotechar="'")
-    lines.drop(columns=["tags"]).to_csv(snakemake.output["lines"], quotechar="'")
+    lines.to_csv(snakemake.output["lines"], quotechar="'")
     links.to_csv(snakemake.output["links"], quotechar="'")
     converters.to_csv(snakemake.output["converters"], quotechar="'")
     transformers.to_csv(snakemake.output["transformers"], quotechar="'")
@@ -1624,3 +1647,7 @@ def build_network(
     links.to_file(snakemake.output["links_geojson"])
     converters.to_file(snakemake.output["converters_geojson"])
     transformers.to_file(snakemake.output["transformers_geojson"])
+
+    # Export polygons for visualisation
+    stations_polygon.to_file(snakemake.output["stations_polygon"])
+    buses_polygon.to_file(snakemake.output["buses_polygon"])