diff --git a/config/config.default.yaml b/config/config.default.yaml index 15af59344..64197dbbe 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -86,7 +86,7 @@ co2_budget: electricity: voltages: [220., 300., 330., 380., 400., 500., 750.] base_network: osm-prebuilt - osm-prebuilt-version: 0.5 + osm-prebuilt-version: 0.6 gaslimit_enable: false gaslimit: false co2limit_enable: false diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 416191c8d..6e87d40b8 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -1,7 +1,7 @@ ,Unit,Values,Description voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer." -osm-prebuilt-version, --, "float, any value in range 0.1-0.5", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." +osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. gaslimit,MWhth,float or false,Global gas usage limit co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks.. diff --git a/doc/data-base-network.rst b/doc/data-base-network.rst index 5a61f71c5..98372c841 100644 --- a/doc/data-base-network.rst +++ b/doc/data-base-network.rst @@ -15,11 +15,11 @@ The map might take a moment to load. To view it in full screen, click `here `) + [Data set]. Zenodo. https://doi.org/10.5281/zenodo.14144752 +- **Link:** https://zenodo.org/records/14144752 +- **License:** ODbL (`reference `) - **Description:** Pre-built data of high-voltage transmission grid in Europe from OpenStreetMap. This dataset contains a topologically connected representation of the European diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 604537fdd..749b96b0b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -103,7 +103,7 @@ Upcoming Release - Single transformers for each combination of voltage level per substation. Transformers now have a capacity s_nom based on connected lines - Use of OSM relations where available and unambiguous (Overwriting all lines that are members of the respective relation to avoid duplicates) -* Updated osm-prebuilt base network to version 0.5, for changelog, see https://zenodo.org/records/13981528 +* Updated osm-prebuilt base network to version 0.6, for changelog, see https://zenodo.org/records/14144752 * Bugfix: vehicle-to-grid dispatch capacity is now limited by the fraction of vehicles participating in demand-side-management, halving the dispatch capacity under the default demand-side management participation rate of 0.5. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 3fcbfed16..db390b9d7 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -815,6 +815,8 @@ if config["electricity"]["base_network"] == "osm-raw": converters_geojson=resources("osm-raw/build/geojson/converters.geojson"), transformers_geojson=resources("osm-raw/build/geojson/transformers.geojson"), substations_geojson=resources("osm-raw/build/geojson/buses.geojson"), + stations_polygon=resources("osm-raw/build/geojson/stations_polygon.geojson"), + buses_polygon=resources("osm-raw/build/geojson/buses_polygon.geojson"), log: logs("build_osm_network.log"), benchmark: diff --git a/rules/development.smk b/rules/development.smk index 465490258..832a5f97b 100644 --- a/rules/development.smk +++ b/rules/development.smk @@ -5,14 +5,19 @@ if config["electricity"]["base_network"] == "osm-raw": rule prepare_osm_network_release: + params: + line_types=config["lines"]["types"], input: base_network=resources("networks/base.nc"), + stations_polygon=resources("osm-raw/build/geojson/stations_polygon.geojson"), + buses_polygon=resources("osm-raw/build/geojson/buses_polygon.geojson"), output: buses=resources("osm-raw/release/buses.csv"), converters=resources("osm-raw/release/converters.csv"), lines=resources("osm-raw/release/lines.csv"), links=resources("osm-raw/release/links.csv"), transformers=resources("osm-raw/release/transformers.csv"), + map=resources("osm-raw/release/map.html"), log: logs("prepare_osm_network_release.log"), benchmark: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 9154386b1..a2d4a9543 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -546,6 +546,7 @@ if config["enable"]["retrieve"] and ( 0.3: "13358976", 0.4: "13759222", 0.5: "13981528", + 0.6: "14144752", } # update rule to use the correct version @@ -566,12 +567,16 @@ if config["enable"]["retrieve"] and ( transformers=storage( f"https://zenodo.org/records/{osm_prebuilt_version[config['electricity']['osm-prebuilt-version']]}/files/transformers.csv" ), + map=storage( + f"https://zenodo.org/records/{osm_prebuilt_version[config['electricity']['osm-prebuilt-version']]}/files/map.html" + ), output: buses=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/buses.csv", converters=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/converters.csv", lines=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/lines.csv", links=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/links.csv", transformers=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/transformers.csv", + map=f"data/osm-prebuilt/{config['electricity']['osm-prebuilt-version']}/map.html", log: "logs/retrieve_osm_prebuilt.log", threads: 1 diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 31884d461..169e74c63 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -1429,7 +1429,7 @@ def _finalise_network(all_buses, converters, lines, links, transformers): lines_all["voltage"] = lines_all["voltage"] / 1000 lines_all["length"] = lines_all["length"].round(2) lines_all["under_construction"] = False - lines_all["tags"] = lines_all["contains_lines"] + lines_all["tags"] = lines_all["contains_lines"].apply(lambda x: ";".join(x)) lines_all["underground"] = lines_all["underground"].replace({True: "t", False: "f"}) lines_all["under_construction"] = lines_all["under_construction"].replace( {True: "t", False: "f"} @@ -1581,12 +1581,33 @@ def build_network( lines["length"] = lines.to_crs(DISTANCE_CRS).length links["length"] = links.to_crs(DISTANCE_CRS).length + # Shapes + stations_polygon = stations[["station_id", "geometry"]].copy() + all_buses_polygon = buses_polygon.copy() + all_buses_polygon["dc"] = False + all_buses_polygon = pd.concat( + [ + all_buses_polygon, + dc_buses[["bus_id", "polygon", "dc"]].rename( + columns={"polygon": "geometry"} + ), + ] + ) + ### Saving outputs to PyPSA-compatible format buses_final, converters_final, lines_final, links_final, transformers_final = ( _finalise_network(all_buses, converters, lines, links, transformers) ) - return buses_final, converters_final, lines_final, links_final, transformers_final + return ( + buses_final, + converters_final, + lines_final, + links_final, + transformers_final, + stations_polygon, + all_buses_polygon, + ) if __name__ == "__main__": @@ -1604,16 +1625,18 @@ def build_network( country_shapes = gpd.read_file(snakemake.input["country_shapes"]).set_index("name") # Build network - buses, converters, lines, links, transformers = build_network( - snakemake.input, - country_shapes, - voltages, - line_types, + buses, converters, lines, links, transformers, stations_polygon, buses_polygon = ( + build_network( + snakemake.input, + country_shapes, + voltages, + line_types, + ) ) # Export to csv for base_network buses.to_csv(snakemake.output["substations"], quotechar="'") - lines.drop(columns=["tags"]).to_csv(snakemake.output["lines"], quotechar="'") + lines.to_csv(snakemake.output["lines"], quotechar="'") links.to_csv(snakemake.output["links"], quotechar="'") converters.to_csv(snakemake.output["converters"], quotechar="'") transformers.to_csv(snakemake.output["transformers"], quotechar="'") @@ -1624,3 +1647,7 @@ def build_network( links.to_file(snakemake.output["links_geojson"]) converters.to_file(snakemake.output["converters_geojson"]) transformers.to_file(snakemake.output["transformers_geojson"]) + + # Export polygons for visualisation + stations_polygon.to_file(snakemake.output["stations_polygon"]) + buses_polygon.to_file(snakemake.output["buses_polygon"]) diff --git a/scripts/prepare_osm_network_release.py b/scripts/prepare_osm_network_release.py index b66d851f7..5233407e5 100644 --- a/scripts/prepare_osm_network_release.py +++ b/scripts/prepare_osm_network_release.py @@ -4,15 +4,19 @@ # SPDX-License-Identifier: MIT import logging -import os -import pandas as pd +import folium +import geopandas as gpd +import numpy as np import pypsa from _helpers import configure_logging, set_scenario_config +from base_network import _get_linetype_by_voltage +from shapely.wkt import loads logger = logging.getLogger(__name__) +GEO_CRS = "EPSG:4326" BUSES_COLUMNS = [ "bus_id", "voltage", @@ -30,10 +34,17 @@ "bus0", "bus1", "voltage", + "i_nom", "circuits", + "s_nom", + "r", + "x", + "b", "length", "underground", "under_construction", + "type", + "tags", "geometry", ] LINKS_COLUMNS = [ @@ -98,6 +109,115 @@ def export_clean_csv(df, columns, output_file): return None +def create_geometries(network, crs=GEO_CRS): + """ + Create GeoDataFrames for different network components with specified coordinate reference system (CRS). + + Parameters: + network (PyPSA Network): The network object containing buses, lines, links, converters, and transformers data. + crs (str, optional): Coordinate reference system to be used for the GeoDataFrames. Defaults to GEO_CRS. + + Returns: + tuple: A tuple containing the following GeoDataFrames: + - buses (GeoDataFrame): GeoDataFrame containing bus data with geometries. + - lines (GeoDataFrame): GeoDataFrame containing line data with geometries. + - links (GeoDataFrame): GeoDataFrame containing link data with geometries. + - converters (GeoDataFrame): GeoDataFrame containing converter data with geometries. + - transformers (GeoDataFrame): GeoDataFrame containing transformer data with geometries. + """ + buses = network.buses.reset_index()[ + [ + "Bus", + "v_nom", + "dc", + "symbol", + "under_construction", + "tags", + "geometry", + ] + ] + buses["geometry"] = buses.geometry.apply(lambda x: loads(x)) + buses = gpd.GeoDataFrame(buses, geometry="geometry", crs=crs) + + lines = network.lines.reset_index()[ + [ + "Line", + "bus0", + "bus1", + "v_nom", + "i_nom", + "num_parallel", + "s_nom", + "r", + "x", + "b", + "length", + "underground", + "under_construction", + "type", + "tags", + "geometry", + ] + ] + # Create shapely linestring from geometry column + lines["geometry"] = lines.geometry.apply(lambda x: loads(x)) + lines = gpd.GeoDataFrame(lines, geometry="geometry", crs=crs) + + links = ( + network.links[~is_converter] + .reset_index() + .rename(columns={"voltage": "v_nom"})[ + [ + "Link", + "bus0", + "bus1", + "v_nom", + "p_nom", + "length", + "underground", + "under_construction", + "tags", + "geometry", + ] + ] + ) + links["geometry"] = links.geometry.apply(lambda x: loads(x)) + links = gpd.GeoDataFrame(links, geometry="geometry", crs=crs) + + converters = ( + network.links[is_converter] + .reset_index() + .rename(columns={"voltage": "v_nom"})[ + [ + "Link", + "bus0", + "bus1", + "v_nom", + "p_nom", + "geometry", + ] + ] + ) + converters["geometry"] = converters.geometry.apply(lambda x: loads(x)) + converters = gpd.GeoDataFrame(converters, geometry="geometry", crs=crs) + + transformers = network.transformers.reset_index()[ + [ + "Transformer", + "bus0", + "bus1", + "voltage_bus0", + "voltage_bus1", + "s_nom", + "geometry", + ] + ] + transformers["geometry"] = transformers.geometry.apply(lambda x: loads(x)) + transformers = gpd.GeoDataFrame(transformers, geometry="geometry", crs=crs) + + return buses, lines, links, converters, transformers + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -107,8 +227,44 @@ def export_clean_csv(df, columns, output_file): configure_logging(snakemake) set_scenario_config(snakemake) + # Params + line_types = snakemake.params.line_types + network = pypsa.Network(snakemake.input.base_network) + logger.info("Re-adding line types to network.") + # Re-add line types + network.lines.loc[:, "type"] = network.lines.v_nom.apply( + lambda x: _get_linetype_by_voltage(x, line_types) + ) + + # Calculate dependent variables (r, x) + logger.info("Calculating dependent variables for network (r, x).") + network.calculate_dependent_values() + + # i_nom + network.lines["i_nom"] = ( + (network.lines.s_nom / network.lines.v_nom / network.lines.num_parallel) + .div(np.sqrt(3)) + .round(3) + ) # kA + + # Rounding of dependent values + network.lines.s_nom = network.lines.s_nom.round(3) + network.lines.r = network.lines.r.round(6) + network.lines.x = network.lines.x.round(6) + network.lines.b = network.lines.b.round(8) + + # Convert v_nom and num_parallel to integers + network.buses.v_nom = network.buses.v_nom.astype(int) + network.lines.v_nom = network.lines.v_nom.astype(int) + network.lines.num_parallel = network.lines.num_parallel.astype(int) + network.links.voltage = network.links.voltage.astype(int) + network.links.p_nom = network.links.p_nom.astype(int) + network.transformers.voltage_bus0 = network.transformers.voltage_bus0.astype(int) + network.transformers.voltage_bus1 = network.transformers.voltage_bus1.astype(int) + network.transformers.s_nom = network.transformers.s_nom.astype(int) + network.buses["dc"] = network.buses.pop("carrier").map({"DC": "t", "AC": "f"}) network.lines.length = network.lines.length * 1e3 network.links.length = network.links.length * 1e3 @@ -153,4 +309,122 @@ def export_clean_csv(df, columns, output_file): network.links[is_converter], CONVERTERS_COLUMNS, snakemake.output.converters ) + ### Create interactive map + buses, lines, links, converters, transformers = create_geometries( + network, crs=GEO_CRS + ) + stations_polygon = gpd.read_file(snakemake.input.stations_polygon) + buses_polygon = gpd.read_file(snakemake.input.buses_polygon) + + # Only keep stations_polygon that contain buses points + stations_polygon = gpd.sjoin( + stations_polygon, buses, how="left", predicate="contains" + ) + stations_polygon = stations_polygon[stations_polygon.index_right.notnull()] + stations_polygon = stations_polygon.drop_duplicates(subset=["station_id"]) + stations_polygon = stations_polygon[["station_id", "geometry"]] + + buses_polygon = gpd.sjoin(buses_polygon, buses, how="left", predicate="contains") + buses_polygon = buses_polygon[buses_polygon.index_right.notnull()] + buses_polygon = buses_polygon.drop_duplicates(subset=["bus_id", "dc_left"]) + buses_polygon.rename(columns={"dc_left": "dc"}, inplace=True) + buses_polygon = buses_polygon[["bus_id", "dc", "geometry"]] + + map = None + map = folium.Map(tiles="CartoDB positron", zoom_start=5, location=[53.5, 10]) + map = stations_polygon.loc[ + ( + stations_polygon.station_id.str.startswith("way") + | stations_polygon.station_id.str.startswith("relation") + ) + ].explore( + color="darkred", + popup=True, + m=map, + name="Clustered substations", + zindex=100, + ) + map = stations_polygon.loc[ + ~( + stations_polygon.station_id.str.startswith("way") + | stations_polygon.station_id.str.startswith("relation") + ) + ].explore( + color="grey", + popup=True, + m=map, + name="Clustered substations (virtual)", + zindex=101, + ) + map = buses_polygon.loc[buses_polygon.dc == False].explore( + color="yellow", + popup=True, + m=map, + name="Buses (AC)", + zindex=102, + ) + map = buses_polygon.loc[buses_polygon.dc == True].explore( + color="grey", + popup=True, + m=map, + name="Buses (DC)", + zindex=103, + ) + map = lines.explore( + color="rosybrown", + popup=True, + m=map, + name="Lines (AC)", + zindex=104, + ) + map = links.explore( + color="darkseagreen", + popup=True, + m=map, + name="Links (DC)", + zindex=105, + ) + map = transformers.explore( + color="orange", + popup=True, + m=map, + name="Transformers", + zindex=106, + ) + map = converters.explore( + color="purple", + popup=True, + m=map, + name="Converters", + zindex=107, + ) + map = buses.loc[buses.dc == "f"].explore( + color="red", + popup=True, + m=map, + name="Buses (AC, Points)", + zindex=108, + ) + map = buses.loc[buses.dc == "t"].explore( + color="black", + popup=True, + m=map, + name="Buses (DC, Points)", + zindex=109, + ) + # Add legend + folium.LayerControl(collapsed=False).add_to(map) + + map_title = "Prebuilt electricity high-voltage grid based on OpenStreetMap data" + map.get_root().html.add_child( + folium.Element( + f"

{map_title}

" + ) + ) + map + + # Export map + logger.info("Exporting interactive map.") + map.save(snakemake.output.map) + logger.info("Export of OSM network for release complete.")