From 0ae016a7d0a236e6c25ddd0044d4b02e623af7b7 Mon Sep 17 00:00:00 2001 From: cpschau Date: Wed, 10 Apr 2024 17:10:56 +0200 Subject: [PATCH 01/57] write shapes to base network --- scripts/base_network.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/base_network.py b/scripts/base_network.py index 346f99a59..f78f4b04d 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -697,6 +697,14 @@ def _adjust_capacities_of_under_construction_branches(n, config): return n +def _set_shapes(n, country_shapes, offshore_shapes): + # Write the geodataframes country_shapes and offshore_shapes to the network.shapes component + country_shapes = gpd.read_file(country_shapes).rename(columns={"name": "idx"}) + country_shapes["type"] = "country" + offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) + offshore_shapes["type"] = "offshore" + all_shapes = pd.concat([country_shapes, offshore_shapes]) + n.shapes = pd.concat([n.shapes, all_shapes]) def base_network( eg_buses, @@ -758,12 +766,16 @@ def base_network( n = _adjust_capacities_of_under_construction_branches(n, config) + _set_shapes(n, country_shapes, offshore_shapes) + return n if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake + import os + os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("base_network") configure_logging(snakemake) From c19b8d760ec19830e75dd2e3999acdbe17ca2866 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 10 Apr 2024 17:47:29 +0200 Subject: [PATCH 02/57] build_bus_regions: add shapes to network --- scripts/build_bus_regions.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 9d993c17f..816c7fad4 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -173,12 +173,31 @@ def voronoi_partition_pts(points, outline): offshore_regions_c = offshore_regions_c.loc[offshore_regions_c.area > 1e-2] offshore_regions.append(offshore_regions_c) - pd.concat(onshore_regions, ignore_index=True).to_file( - snakemake.output.regions_onshore + gdf = pd.concat(onshore_regions, ignore_index=True) + gdf.to_file(snakemake.output.regions_onshore) + + index = gdf.index.astype(int) + n.shapes.index.astype(int).max() + 1 + n.madd( + "Shape", + index, + geometry=gdf.geometry, + idx=index, + component="Bus", + type="onshore", ) if offshore_regions: - pd.concat(offshore_regions, ignore_index=True).to_file( - snakemake.output.regions_offshore + gdf = pd.concat(offshore_regions, ignore_index=True) + gdf.to_file(snakemake.output.regions_offshore) + + index = gdf.index.astype(int) + n.shapes.index.astype(int).max() + 1 + n.madd( + "Shape", + index, + geometry=gdf.geometry, + idx=index, + component="Bus", + type="offshore", ) + else: offshore_shapes.to_frame().to_file(snakemake.output.regions_offshore) From 7d0b775ca9c84ead87cafc547dca5b2d28f6c824 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:48:44 +0000 Subject: [PATCH 03/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/base_network.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index f78f4b04d..77d3d2507 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -697,6 +697,7 @@ def _adjust_capacities_of_under_construction_branches(n, config): return n + def _set_shapes(n, country_shapes, offshore_shapes): # Write the geodataframes country_shapes and offshore_shapes to the network.shapes component country_shapes = gpd.read_file(country_shapes).rename(columns={"name": "idx"}) @@ -704,7 +705,8 @@ def _set_shapes(n, country_shapes, offshore_shapes): offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) offshore_shapes["type"] = "offshore" all_shapes = pd.concat([country_shapes, offshore_shapes]) - n.shapes = pd.concat([n.shapes, all_shapes]) + n.shapes = pd.concat([n.shapes, all_shapes]) + def base_network( eg_buses, @@ -773,8 +775,10 @@ def base_network( if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake import os + + from _helpers import mock_snakemake + os.chdir(os.path.dirname(os.path.abspath(__file__))) snakemake = mock_snakemake("base_network") From 9686407756684b064f27f77b75d06e4d3149142e Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 10 Apr 2024 17:58:08 +0200 Subject: [PATCH 04/57] cluster_network: add regions to n.shapes --- scripts/cluster_network.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index b0b73ade4..a2473beb1 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -428,16 +428,31 @@ def clustering_for_n_clusters( return clustering -def cluster_regions(busmaps, input=None, output=None): +def cluster_regions(busmaps, which, input=None, output=None): busmap = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) - for which in ("regions_onshore", "regions_offshore"): - regions = gpd.read_file(getattr(input, which)) - regions = regions.reindex(columns=["name", "geometry"]).set_index("name") - regions_c = regions.dissolve(busmap) - regions_c.index.name = "name" - regions_c = regions_c.reset_index() - regions_c.to_file(getattr(output, which)) + regions = gpd.read_file(getattr(input, which)) + regions = regions.reindex(columns=["name", "geometry"]).set_index("name") + regions_c = regions.dissolve(busmap) + regions_c.index.name = "name" + regions_c = regions_c.reset_index() + regions_c.to_file(getattr(output, which)) + + # remove old regions + remove = n.shapes.query("component == 'Bus' and type == @which").index + n.mremove("Shape", remove) + + # add new clustered regions + index = regions_c.index.astype(int) + n.shapes.index.astype(int).max() + 1 + type = which.split("_")[1] + n.madd( + "Shape", + index, + geometry=regions_c.geometry, + idx=index, + component="Bus", + type="which", + ) def plot_busmap_for_n_clusters(n, n_clusters, fn=None): @@ -555,4 +570,5 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): ): # also available: linemap_positive, linemap_negative getattr(clustering, attr).to_csv(snakemake.output[attr]) - cluster_regions((clustering.busmap,), snakemake.input, snakemake.output) + for which in ["regions_onshore", "regions_offshore"]: + cluster_regions((clustering.busmap,), which, snakemake.input, snakemake.output) From 47134a88d833e2db1c1a6d95fd070ab208d5d5d6 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 10 Apr 2024 18:13:14 +0200 Subject: [PATCH 05/57] regions to n.shapes: smooth out remaining issues --- scripts/base_network.py | 2 +- scripts/build_bus_regions.py | 7 +++++-- scripts/cluster_network.py | 3 ++- scripts/simplify_network.py | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 77d3d2507..528f04bfa 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -705,7 +705,7 @@ def _set_shapes(n, country_shapes, offshore_shapes): offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) offshore_shapes["type"] = "offshore" all_shapes = pd.concat([country_shapes, offshore_shapes]) - n.shapes = pd.concat([n.shapes, all_shapes]) + n.shapes = pd.concat([n.shapes, all_shapes], ignore_index=True) def base_network( diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 816c7fad4..d26e6ba6d 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -176,7 +176,8 @@ def voronoi_partition_pts(points, outline): gdf = pd.concat(onshore_regions, ignore_index=True) gdf.to_file(snakemake.output.regions_onshore) - index = gdf.index.astype(int) + n.shapes.index.astype(int).max() + 1 + offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 + index = gdf.index.astype(int) + offset n.madd( "Shape", index, @@ -185,11 +186,13 @@ def voronoi_partition_pts(points, outline): component="Bus", type="onshore", ) + if offshore_regions: gdf = pd.concat(offshore_regions, ignore_index=True) gdf.to_file(snakemake.output.regions_offshore) - index = gdf.index.astype(int) + n.shapes.index.astype(int).max() + 1 + offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 + index = gdf.index.astype(int) + offset n.madd( "Shape", index, diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index a2473beb1..7a4976262 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -443,7 +443,8 @@ def cluster_regions(busmaps, which, input=None, output=None): n.mremove("Shape", remove) # add new clustered regions - index = regions_c.index.astype(int) + n.shapes.index.astype(int).max() + 1 + offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 + index = regions_c.index.astype(int) + offset type = which.split("_")[1] n.madd( "Shape", diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 24df7312e..6e4041920 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -629,4 +629,5 @@ def cluster( busmap_s = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) busmap_s.to_csv(snakemake.output.busmap) - cluster_regions(busmaps, snakemake.input, snakemake.output) + for which in ["regions_onshore", "regions_offshore"]: + cluster_regions(busmaps, which, snakemake.input, snakemake.output) From 84de226cb01a59945ab3505ad20fce62a492fa79 Mon Sep 17 00:00:00 2001 From: chrstphtrs Date: Thu, 11 Apr 2024 09:43:46 +0200 Subject: [PATCH 06/57] Add documentation section for how to contribute documentation --- doc/contributing.rst | 13 +++++++++++++ doc/release_notes.rst | 2 ++ 2 files changed, 15 insertions(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index 0bf969b6e..2d0253826 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -35,3 +35,16 @@ including our own are reviewed by a second person before they are incorporated i If you are unfamiliar with pull requests, the GitHub help pages have a nice `guide `_. To ask and answer general usage questions, join the `PyPSA mailing list `_. + +Contributing to the documentation +==================================== + +We strive to keep documentation useful and up to date for all PyPSA users. If you encounter an area where documentation is not available or insufficient, we very much welcome your contribution. Here is How To: + +#. Install the conda environment for documentation from the `PyPSA repository `_. + (Here is `how to install a conda environment `_.) +#. Make your changes in the corresponding .rst file under ``pypsa-eur/doc``. +#. Compile your changes by running the following command in your terminal in the ``doc`` folder: ``make html`` + You may encounter some warnings, but end up with a message such as ``build succeeded, XX warnings.``. html files to review your changes can then be found under ``doc/_build/html``. +#. Contribute your documentation in a pull request (`here is a guide `_). + \ No newline at end of file diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b782a6dfa..c50999436 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -176,6 +176,8 @@ Upcoming Release * Fix custom busmap read in `cluster_network`. +* Add documentation section for how to contribute documentation + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== From 73971199d2f6416f219281b1823ecf39784ebffd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:46:20 +0000 Subject: [PATCH 07/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 2d0253826..52dc8280b 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -47,4 +47,4 @@ We strive to keep documentation useful and up to date for all PyPSA users. If yo #. Compile your changes by running the following command in your terminal in the ``doc`` folder: ``make html`` You may encounter some warnings, but end up with a message such as ``build succeeded, XX warnings.``. html files to review your changes can then be found under ``doc/_build/html``. #. Contribute your documentation in a pull request (`here is a guide `_). - \ No newline at end of file + From 590d3635790222ddddefc388a59547b4db64d02c Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 11 Apr 2024 12:13:07 +0200 Subject: [PATCH 08/57] clustering: add docstring to cluster_regions function, fix network reference --- scripts/cluster_network.py | 35 ++++++++++++++++++++++++----------- scripts/simplify_network.py | 2 +- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 7a4976262..a18a20799 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -428,7 +428,20 @@ def clustering_for_n_clusters( return clustering -def cluster_regions(busmaps, which, input=None, output=None): +def cluster_regions(n, busmaps, which, input=None, output=None): + """ + Cluster regions based on busmaps and save the results to a file and to the + network. + + Parameters: + - busmaps (list): A list of busmaps used for clustering. + - which (str): The type of regions to cluster. + - input (str, optional): The input file path. Defaults to None. + - output (str, optional): The output file path. Defaults to None. + + Returns: + None + """ busmap = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) regions = gpd.read_file(getattr(input, which)) @@ -438,7 +451,7 @@ def cluster_regions(busmaps, which, input=None, output=None): regions_c = regions_c.reset_index() regions_c.to_file(getattr(output, which)) - # remove old regions + # remove original regions remove = n.shapes.query("component == 'Bus' and type == @which").index n.mremove("Shape", remove) @@ -456,8 +469,8 @@ def cluster_regions(busmaps, which, input=None, output=None): ) -def plot_busmap_for_n_clusters(n, n_clusters, fn=None): - busmap = busmap_for_n_clusters(n, n_clusters) +def plot_busmap_for_n_clusters(n, n_clusters, solver_name="scip", fn=None): + busmap = busmap_for_n_clusters(n, n_clusters, solver_name) cs = busmap.unique() cr = sns.color_palette("hls", len(cs)) n.plot(bus_colors=busmap.map(dict(zip(cs, cr)))) @@ -554,17 +567,15 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): params.focus_weights, ) - update_p_nom_max(clustering.network) + nc = clustering.network + update_p_nom_max(nc) if params.cluster_network.get("consider_efficiency_classes"): labels = [f" {label} efficiency" for label in ["low", "medium", "high"]] - nc = clustering.network nc.generators["carrier"] = nc.generators.carrier.replace(labels, "", regex=True) - clustering.network.meta = dict( - snakemake.config, **dict(wildcards=dict(snakemake.wildcards)) - ) - clustering.network.export_to_netcdf(snakemake.output.network) + nc.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + nc.export_to_netcdf(snakemake.output.network) for attr in ( "busmap", "linemap", @@ -572,4 +583,6 @@ def plot_busmap_for_n_clusters(n, n_clusters, fn=None): getattr(clustering, attr).to_csv(snakemake.output[attr]) for which in ["regions_onshore", "regions_offshore"]: - cluster_regions((clustering.busmap,), which, snakemake.input, snakemake.output) + cluster_regions( + nc, (clustering.busmap,), which, snakemake.input, snakemake.output + ) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 6e4041920..7b8710a05 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -630,4 +630,4 @@ def cluster( busmap_s.to_csv(snakemake.output.busmap) for which in ["regions_onshore", "regions_offshore"]: - cluster_regions(busmaps, which, snakemake.input, snakemake.output) + cluster_regions(n, busmaps, which, snakemake.input, snakemake.output) From 859212b21ff83e35a414d6874e7abba6e492b947 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 11 Apr 2024 06:41:59 +0000 Subject: [PATCH 09/57] Use powerplantmatching IRENASTAT for add_existing_baseyear --- config/config.default.yaml | 1 - .../offwind_capacity_IRENA.csv | 34 ------ .../onwind_capacity_IRENA.csv | 34 ------ .../solar_capacity_IRENA.csv | 34 ------ doc/configtables/enable.csv | 1 - doc/retrieve.rst | 5 - rules/retrieve.smk | 18 --- rules/solve_myopic.smk | 3 - rules/solve_perfect.smk | 3 - scripts/add_existing_baseyear.py | 29 +++-- scripts/retrieve_irena.py | 108 ------------------ 11 files changed, 20 insertions(+), 250 deletions(-) delete mode 100644 data/existing_infrastructure/offwind_capacity_IRENA.csv delete mode 100644 data/existing_infrastructure/onwind_capacity_IRENA.csv delete mode 100644 data/existing_infrastructure/solar_capacity_IRENA.csv delete mode 100644 scripts/retrieve_irena.py diff --git a/config/config.default.yaml b/config/config.default.yaml index d438c51f7..94c911863 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -72,7 +72,6 @@ enable: retrieve_sector_databundle: true retrieve_cost_data: true build_cutout: false - retrieve_irena: false retrieve_cutout: true build_natura_raster: false retrieve_natura_raster: true diff --git a/data/existing_infrastructure/offwind_capacity_IRENA.csv b/data/existing_infrastructure/offwind_capacity_IRENA.csv deleted file mode 100644 index d2a3f0f10..000000000 --- a/data/existing_infrastructure/offwind_capacity_IRENA.csv +++ /dev/null @@ -1,34 +0,0 @@ -Country/area,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022 -Albania,,,,,,,,,,,,,,,,,,,,,,, -Austria,,,,,,,,,,,,,,,,,,,,,,, -Belgium,,,,,,,,,,31.5,196.5,196.5,381.0,707.7,707.7,712.0,712.2,877.2,1185.9,1555.5,2261.8,2261.8,2261.8 -Bosnia Herzg,,,,,,,,,,,,,,,,,,,,,,, -Bulgaria,,,,,,,,,,,,,,,,,,,,,,, -Croatia,,,,,,,,,,,,,,,,,,,,,,, -Czechia,,,,,,,,,,,,,,,,,,,,,,, -Denmark,49.95,49.95,213.95,423.35,423.35,423.35,423.35,423.35,423.35,660.85,867.85,871.45,921.85,1271.05,1271.05,1271.05,1271.05,1263.8,1700.8,1700.8,1700.8,2305.6,2305.6 -Estonia,,,,,,,,,,,,,,,,,,,,,,, -Finland,,,,,,,,,24.0,24.0,26.3,26.3,26.3,26.3,26.3,32.0,32.0,72.7,72.7,73.0,73.0,73.0,73.0 -France,,,,,,,,,,,,,,,,,,2.0,2.0,2.0,2.0,2.0,482.0 -Germany,,,,,,,,,,35.0,80.0,188.0,268.0,508.0,994.0,3283.0,4132.0,5406.0,6393.0,7555.0,7787.0,7787.0,8129.0 -Greece,,,,,,,,,,,,,,,,,,,,,,, -Hungary,,,,,,,,,,,,,,,,,,,,,,, -Ireland,,,,,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2,25.2 -Italy,,,,,,,,,,,,,,,,,,,,,,,30.0 -Latvia,,,,,,,,,,,,,,,,,,,,,,, -Lithuania,,,,,,,,,,,,,,,,,,,,,,, -Luxembourg,,,,,,,,,,,,,,,,,,,,,,, -Montenegro,,,,,,,,,,,,,,,,,,,,,,, -Netherlands,,,,,,,108.0,108.0,228.0,228.0,228.0,228.0,228.0,228.0,228.0,357.0,957.0,957.0,957.0,957.0,2459.5,2459.5,2571.0 -North Macedonia,,,,,,,,,,,,,,,,,,,,,,, -Norway,,,,,,,,,,2.3,2.3,2.3,2.3,2.3,2.3,2.3,2.3,2.3,2.3,2.3,2.3,6.3,66.3 -Poland,,,,,,,,,,,,,,,,,,,,,,, -Portugal,,,,,,,,,,,,1.86,2.0,2.0,2.0,2.0,,,,,25.0,25.0,25.0 -Romania,,,,,,,,,,,,,,,,,,,,,,, -Serbia,,,,,,,,,,,,,,,,,,,,,,, -Slovakia,,,,,,,,,,,,,,,,,,,,,,, -Slovenia,,,,,,,,,,,,,,,,,,,,,,, -Spain,,,,,,,,,,,,,,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0 -Sweden,13.0,22.0,22.0,22.0,22.0,22.0,22.0,131.0,133.0,163.0,163.0,163.0,163.0,212.0,213.0,213.0,203.0,203.0,203.0,203.0,203.0,193.0,193.0 -Switzerland,,,,,,,,,,,,,,,,,,,,,,, -UK,4.0,4.0,4.0,64.0,124.0,214.0,304.0,394.0,596.2,951.0,1341.0,1838.0,2995.0,3696.0,4501.0,5093.0,5293.0,6988.0,8181.0,9888.0,10383.0,11255.0,13928.0 diff --git a/data/existing_infrastructure/onwind_capacity_IRENA.csv b/data/existing_infrastructure/onwind_capacity_IRENA.csv deleted file mode 100644 index cd5ac19c0..000000000 --- a/data/existing_infrastructure/onwind_capacity_IRENA.csv +++ /dev/null @@ -1,34 +0,0 @@ -Country/area,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022 -Albania,,,,,,,,,,,,,,,,,,,,,,, -Austria,50.0,67.0,109.0,322.0,581.0,825.22,968.27,991.16,991.97,1000.99,1015.83,1105.97,1337.15,1674.54,2110.28,2488.73,2730.0,2886.7,3132.71,3224.12,3225.98,3407.81,3735.81 -Belgium,14.0,26.0,31.0,67.0,96.0,167.0,212.0,276.0,324.0,576.5,715.5,872.5,985.9,1061.3,1225.0,1469.3,1621.6,1902.2,2119.0,2308.0,2410.9,2686.6,2989.6 -Bosnia Herzg,,,,,,,,,,,,0.3,0.3,0.3,0.3,0.3,0.3,0.3,51.0,87.0,87.0,135.0,135.0 -Bulgaria,,,,,1.0,8.0,27.0,30.0,114.0,333.0,488.0,541.0,677.0,683.0,699.0,699.0,699.0,698.39,698.92,703.12,702.8,704.38,704.38 -Croatia,,,,,6.0,6.0,17.0,17.0,17.0,70.0,79.0,130.0,180.0,254.0,339.0,418.0,483.0,576.1,586.3,646.3,801.3,986.9,1042.9 -Czechia,2.0,,6.4,10.6,16.5,22.0,43.5,113.8,150.0,193.0,213.0,213.0,258.0,262.0,278.0,281.0,282.0,308.21,316.2,339.41,339.42,339.41,339.41 -Denmark,2340.07,2447.2,2680.58,2696.57,2700.36,2704.49,2712.35,2700.86,2739.52,2821.24,2933.98,3080.53,3240.09,3547.87,3615.35,3805.92,3974.09,4225.15,4421.86,4409.74,4566.23,4715.24,4782.24 -Estonia,,,1.0,3.0,7.0,31.0,31.0,50.0,77.0,104.0,108.0,180.0,266.0,248.0,275.0,300.0,310.0,311.8,310.0,316.0,317.0,315.0,315.0 -Finland,38.0,39.0,43.0,52.0,82.0,82.0,86.0,110.0,119.0,123.0,170.7,172.7,230.7,420.7,600.7,973.0,1533.0,1971.3,1968.3,2211.0,2513.0,3184.0,5541.0 -France,38.0,66.0,138.0,218.0,358.0,690.0,1412.0,2223.0,3403.0,4582.0,5912.0,6758.02,7607.5,8155.96,9201.42,10298.18,11566.56,13497.35,14898.14,16424.85,17512.0,18737.98,20637.98 -Germany,6095.0,8754.0,12001.0,14381.0,16419.0,18248.0,20474.0,22116.0,22794.0,25697.0,26823.0,28524.0,30711.0,32969.0,37620.0,41297.0,45303.0,50174.0,52328.0,53187.0,54414.0,56046.0,58165.0 -Greece,226.0,270.0,287.0,371.0,470.0,491.0,749.0,846.0,1022.0,1171.0,1298.0,1640.0,1753.0,1809.0,1978.0,2091.0,2370.0,2624.0,2877.5,3589.0,4119.25,4649.13,4879.13 -Hungary,,1.0,1.0,3.0,3.0,17.0,33.0,61.0,134.0,203.0,293.0,331.0,325.0,329.0,329.0,329.0,329.0,329.0,329.0,323.0,323.0,324.0,324.0 -Ireland,116.5,122.9,134.8,210.3,311.2,468.1,651.3,715.3,917.1,1226.1,1365.2,1559.4,1679.15,1898.1,2258.05,2425.95,2776.45,3293.95,3648.65,4101.25,4281.5,4313.84,4593.84 -Italy,363.0,664.0,780.0,874.0,1127.0,1635.0,1902.0,2702.0,3525.0,4879.0,5794.0,6918.0,8102.0,8542.0,8683.0,9137.0,9384.0,9736.58,10230.25,10679.46,10870.62,11253.73,11749.73 -Latvia,2.0,2.0,22.0,26.0,26.0,26.0,26.0,26.0,28.0,29.0,30.0,36.0,59.0,65.89,68.92,68.17,69.91,77.11,78.17,78.07,78.07,77.13,136.13 -Lithuania,,,,,1.0,1.0,31.0,47.0,54.0,98.0,133.0,202.0,275.0,279.0,288.0,436.0,509.0,518.0,533.0,534.0,540.0,671.0,814.0 -Luxembourg,14.0,13.9,13.9,20.5,34.9,34.9,34.9,34.9,42.92,42.93,43.73,44.53,58.33,58.33,58.34,63.79,119.69,119.69,122.89,135.79,152.74,136.44,165.44 -Montenegro,,,,,,,,,,,,,,,,,,72.0,72.0,118.0,118.0,118.0,118.0 -Netherlands,447.0,486.0,672.0,905.0,1075.0,1224.0,1453.0,1641.0,1921.0,1994.0,2009.0,2088.0,2205.0,2485.0,2637.0,3033.84,3300.12,3245.0,3436.11,3527.16,4188.38,5309.87,6176.0 -North Macedonia,,,,,,,,,,,,,,,37.0,37.0,37.0,37.0,37.0,37.0,37.0,37.0,37.0 -Norway,13.0,13.0,97.0,97.0,152.0,265.0,284.0,348.0,395.0,420.7,422.7,509.7,702.7,815.7,856.7,864.7,880.7,1204.7,1707.7,2911.7,4027.7,5042.7,5067.7 -Poland,4.0,19.0,32.0,35.0,40.0,121.0,172.0,306.0,526.0,709.0,1108.0,1800.0,2564.0,3429.0,3836.0,4886.0,5747.0,5759.36,5766.08,5837.76,6298.25,6967.34,7987.34 -Portugal,83.0,125.0,190.0,268.0,553.0,1064.0,1681.0,2201.0,2857.0,3326.0,3796.0,4254.35,4409.55,4607.95,4854.56,4934.84,5124.1,5124.1,5172.36,5222.75,5097.26,5402.33,5430.33 -Romania,,,,,,1.0,1.0,3.0,5.0,15.0,389.0,988.0,1822.0,2773.0,3244.0,3130.0,3025.0,3029.8,3032.26,3037.52,3012.53,3014.96,3014.96 -Serbia,,,,,,,,,,,,,0.5,0.5,0.5,10.4,17.0,25.0,227.0,398.0,398.0,398.0,398.0 -Slovakia,,,,3.0,3.0,5.0,5.0,5.0,5.0,3.0,3.0,3.0,3.0,5.0,3.0,3.0,3.0,4.0,3.0,4.0,4.0,4.0,4.0 -Slovenia,,,,,,,,,,,,,2.0,2.0,3.0,3.0,3.0,3.3,3.3,3.3,3.3,3.33,3.33 -Spain,2206.0,3397.0,4891.0,5945.0,8317.0,9918.0,11722.0,14820.0,16555.0,19176.0,20693.0,21529.0,22789.0,22953.0,22920.0,22938.0,22985.0,23119.48,23400.06,25585.08,26814.19,27902.65,29302.84 -Sweden,196.0,273.0,335.0,395.0,453.0,500.0,563.0,692.0,956.0,1312.0,1854.0,2601.0,3443.0,3982.0,4875.0,5606.0,6232.0,6408.0,7097.0,8478.0,9773.0,11923.0,14364.0 -Switzerland,3.0,5.0,5.0,5.0,9.0,12.0,12.0,12.0,14.0,18.0,42.0,46.0,49.0,60.0,60.0,60.0,75.0,75.0,75.0,75.0,87.0,87.0,87.0 -UK,431.0,490.0,531.0,678.0,809.0,1351.0,1651.0,2083.0,2849.8,3468.0,4080.0,4758.0,6035.0,7586.0,8573.0,9212.0,10833.0,12597.0,13425.0,13999.0,14075.0,14492.0,14832.0 diff --git a/data/existing_infrastructure/solar_capacity_IRENA.csv b/data/existing_infrastructure/solar_capacity_IRENA.csv deleted file mode 100644 index 01683f8dc..000000000 --- a/data/existing_infrastructure/solar_capacity_IRENA.csv +++ /dev/null @@ -1,34 +0,0 @@ -Country/area,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022 -Albania,,0.1,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.3,0.4,0.56,0.68,0.76,0.87,1.05,1.0,1.0,1.0,14.0,21.0,23.0,28.6 -Austria,5.0,7.0,9.0,23.0,27.0,18.49,19.61,21.42,27.0,45.56,85.27,169.88,333.09,620.78,779.76,931.56,1089.53,1262.01,1447.94,1694.4,2034.74,2773.91,3538.91 -Belgium,,,1.0,1.0,1.0,2.0,2.0,20.0,62.0,386.0,1006.6,1978.6,2646.6,2901.6,3015.0,3131.6,3328.8,3620.6,4000.0,4636.6,5572.8,6012.4,6898.4 -Bosnia Herzg,,,,0.1,0.2,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.35,1.34,7.17,8.17,14.12,16.0,18.15,22.35,34.89,56.51,107.47 -Bulgaria,,,,,,,,0.03,0.1,2.0,25.0,154.0,921.99,1038.54,1028.92,1027.89,1029.89,1030.7,1033.06,1044.39,1100.21,1274.71,1948.36 -Croatia,,,,,,,,,,0.3,0.3,0.3,4.0,19.0,33.0,47.8,55.8,60.0,67.7,84.8,108.5,138.3,182.3 -Czechia,0.1,0.1,0.2,0.3,0.4,0.59,0.84,3.96,39.5,464.6,1727.0,1913.0,2022.0,2063.5,2067.4,2074.9,2067.9,2075.44,2081.05,2110.67,2171.96,2246.09,2627.09 -Denmark,1.0,1.0,2.0,2.0,2.0,3.0,3.0,3.0,3.0,5.0,7.0,17.0,402.0,571.0,607.0,782.11,850.95,906.35,998.0,1080.0,1304.29,1704.04,3122.04 -Estonia,,,,,,,,,,0.1,0.1,0.2,0.38,1.5,3.34,6.5,10.0,15.0,31.9,120.6,207.67,394.77,534.77 -Finland,2.0,3.0,3.0,3.0,4.0,4.0,5.0,5.0,6.0,6.0,7.0,7.0,8.0,9.0,11.0,17.0,39.0,82.0,140.0,222.0,318.0,425.0,590.6 -France,7.0,7.0,8.0,9.0,11.0,13.0,15.0,26.0,80.0,277.0,1044.0,3003.57,4358.75,5277.29,6034.42,7137.52,7702.08,8610.44,9638.88,10738.39,11812.2,14436.97,17036.97 -Germany,114.0,195.0,260.0,435.0,1105.0,2056.0,2899.0,4170.0,6120.0,10564.0,18004.0,25914.0,34075.0,36708.0,37898.0,39222.0,40677.0,42291.0,45156.0,48912.0,53669.0,59371.0,66662.0 -Greece,,1.0,1.0,1.0,1.0,1.0,5.0,9.0,12.0,46.0,202.0,612.0,1536.0,2579.0,2596.0,2604.0,2604.0,2605.53,2651.57,2833.79,3287.72,4277.42,5557.42 -Hungary,,,,,,,,0.4,1.0,1.0,2.0,4.0,12.0,35.0,89.0,172.0,235.0,344.0,728.0,1400.0,2131.0,2968.0,2988.0 -Ireland,,,,,,,,,,,,,,,,,,,,,,, -Italy,19.0,20.0,22.0,26.0,31.0,34.0,45.0,110.0,483.0,1264.0,3592.0,13131.0,16785.0,18185.0,18594.0,18901.0,19283.0,19682.29,20107.59,20865.28,21650.04,22594.26,25076.56 -Latvia,,,,,,,,,,,,,,,,,0.69,0.69,1.96,3.3,5.1,7.16,56.16 -Lithuania,,,,,,,,,0.1,0.1,0.1,0.3,7.0,68.0,69.0,69.0,70.0,70.08,72.0,73.0,80.0,84.0,397.0 -Luxembourg,,0.16,1.59,14.17,23.56,23.58,23.7,23.93,24.56,26.36,29.45,40.67,74.65,95.02,109.93,116.27,121.9,128.1,130.62,159.74,186.64,277.16,319.16 -Montenegro,,,,,,,,,,,,,,,,,,,,,2.57,2.57,22.2 -Netherlands,13.0,21.0,26.0,46.0,50.0,51.0,53.0,54.0,59.0,69.0,90.0,149.0,287.0,650.0,1007.0,1526.26,2135.02,2910.89,4608.0,7226.0,11108.43,14910.69,18848.69 -North Macedonia,,,,,,,,,,,,2.0,4.0,7.0,15.0,17.0,16.7,16.7,16.7,16.71,84.93,84.93,84.93 -Norway,6.0,6.0,6.0,7.0,7.0,7.0,8.0,8.0,8.3,8.7,9.1,9.5,10.0,11.0,13.0,15.0,26.7,44.9,53.11,102.53,141.53,186.53,302.53 -Poland,,,,,,,,,,,,1.11,1.3,2.39,27.15,107.78,187.25,287.09,561.98,1539.26,3954.96,7415.52,11166.52 -Portugal,1.0,1.0,1.0,2.0,2.0,2.0,3.0,24.0,59.0,115.0,134.0,169.6,235.6,293.6,412.6,441.75,493.05,539.42,617.85,832.74,1010.07,1474.78,2364.78 -Romania,,,,,,,,,0.1,0.1,0.1,1.0,41.0,761.0,1293.0,1326.0,1372.0,1374.13,1385.82,1397.71,1382.54,1393.92,1413.92 -Serbia,,,,,,0.1,0.2,0.4,0.9,1.2,1.3,1.5,3.1,4.7,6.0,9.0,11.0,10.0,11.0,11.0,11.5,11.94,11.94 -Slovakia,,,,,,,,,,,19.0,496.0,513.0,533.0,533.0,533.0,533.0,528.0,472.0,590.0,535.0,537.0,537.0 -Slovenia,1.0,1.0,,,,0.05,0.19,0.59,1.0,4.0,12.0,57.0,142.0,187.0,223.0,238.0,233.0,246.8,246.8,277.88,369.78,461.16,632.16 -Spain,1.0,3.0,6.0,10.0,19.0,37.0,113.0,476.0,3365.0,3403.0,3851.0,4260.0,4545.0,4665.0,4672.0,4677.0,4687.0,4696.0,4730.7,8772.02,10100.42,13678.4,18176.73 -Sweden,3.0,3.0,3.0,4.0,4.0,4.0,5.0,6.0,8.0,9.0,11.0,12.0,24.0,43.0,60.0,104.0,153.0,231.0,411.0,698.0,1090.0,1587.0,2587.0 -Switzerland,16.0,18.0,20.0,22.0,24.0,28.0,30.0,37.0,49.0,79.0,125.0,223.0,437.0,756.0,1061.0,1394.0,1664.0,1906.0,2173.0,2498.0,2973.0,3655.0,4339.92 -UK,2.0,3.0,4.0,6.0,8.0,11.0,14.0,18.0,23.0,27.0,95.0,1000.0,1753.0,2937.0,5528.0,9601.0,11914.0,12760.0,13059.0,13345.0,13579.0,13965.0,14660.0 diff --git a/doc/configtables/enable.csv b/doc/configtables/enable.csv index 1e5571b36..06397fdf4 100644 --- a/doc/configtables/enable.csv +++ b/doc/configtables/enable.csv @@ -5,7 +5,6 @@ retrieve_databundle,bool,"{true, false}","Switch to retrieve databundle from zen retrieve_sector_databundle,bool,"{true, false}","Switch to retrieve sector databundle from zenodo via the rule :mod:`retrieve_sector_databundle` or whether to keep a custom databundle located in the corresponding folder." retrieve_cost_data,bool,"{true, false}","Switch to retrieve technology cost data from `technology-data repository `_." build_cutout,bool,"{true, false}","Switch to enable the building of cutouts via the rule :mod:`build_cutout`." -retrieve_irena,bool,"{true, false}",Switch to enable the retrieval of ``existing_capacities`` from IRENASTAT with :mod:`retrieve_irena`. retrieve_cutout,bool,"{true, false}","Switch to enable the retrieval of cutouts from zenodo with :mod:`retrieve_cutout`." build_natura_raster,bool,"{true, false}","Switch to enable the creation of the raster ``natura.tiff`` via the rule :mod:`build_natura_raster`." retrieve_natura_raster,bool,"{true, false}","Switch to enable the retrieval of ``natura.tiff`` from zenodo with :mod:`retrieve_natura_raster`." diff --git a/doc/retrieve.rst b/doc/retrieve.rst index f9d6e2a7d..36163eae5 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -118,11 +118,6 @@ This rule downloads techno-economic assumptions from the `technology-data reposi - ``resources/costs.csv`` -Rule ``retrieve_irena`` -================================ - -.. automodule:: retrieve_irena - Rule ``retrieve_ship_raster`` ================================ diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4b2444834..d0345f36c 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -42,24 +42,6 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", "../scripts/retrieve_databundle.py" -if config["enable"].get("retrieve_irena"): - - rule retrieve_irena: - output: - offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv", - onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", - solar="data/existing_infrastructure/solar_capacity_IRENA.csv", - log: - "logs/retrieve_irena.log", - resources: - mem_mb=1000, - retries: 2 - conda: - "../envs/retrieve.yaml" - script: - "../scripts/retrieve_irena.py" - - if config["enable"]["retrieve"] and config["enable"].get("retrieve_cutout", True): rule retrieve_cutout: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index fe291c6d2..6220af2a8 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -26,9 +26,6 @@ rule add_existing_baseyear: existing_heating_distribution=resources( "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), - existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", - existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", - existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv", output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 70acc8306..51cb39207 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -25,9 +25,6 @@ rule add_existing_baseyear: "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", - existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", - existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv", output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f445efe34..19eea8a5a 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -13,6 +13,7 @@ import country_converter as coco import numpy as np import pandas as pd +import powerplantmatching as pm import pypsa import xarray as xr from _helpers import ( @@ -60,14 +61,22 @@ def add_existing_renewables(df_agg, costs): Append existing renewables to the df_agg pd.DataFrame with the conventional power plants. """ - carriers = {"solar": "solar", "onwind": "onwind", "offwind": "offwind-ac"} + tech_map = {"solar": "PV", "onwind": "Onshore", "offwind": "Offshore"} - for tech in ["solar", "onwind", "offwind"]: - carrier = carriers[tech] + countries = snakemake.config["countries"] + irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() + irena = irena.query("Country in @countries") + irena = irena.groupby(["Technology", "Country", "Year"]).Capacity.sum() - df = pd.read_csv(snakemake.input[f"existing_{tech}"], index_col=0).fillna(0.0) + irena = irena.unstack().reset_index() + + for carrier, tech in tech_map.items(): + df = ( + irena[irena.Technology.str.contains(tech)] + .drop(columns=["Technology"]) + .set_index("Country") + ) df.columns = df.columns.astype(int) - df.index = cc.convert(df.index, to="iso2") # calculate yearly differences df.insert(loc=0, value=0.0, column="1999") @@ -97,14 +106,16 @@ def add_existing_renewables(df_agg, costs): for year in nodal_df.columns: for node in nodal_df.index: - name = f"{node}-{tech}-{year}" + name = f"{node}-{carrier}-{year}" capacity = nodal_df.loc[node, year] if capacity > 0.0: - df_agg.at[name, "Fueltype"] = tech + df_agg.at[name, "Fueltype"] = carrier df_agg.at[name, "Capacity"] = capacity df_agg.at[name, "DateIn"] = year - df_agg.at[name, "lifetime"] = costs.at[tech, "lifetime"] - df_agg.at[name, "DateOut"] = year + costs.at[tech, "lifetime"] - 1 + df_agg.at[name, "lifetime"] = costs.at[carrier, "lifetime"] + df_agg.at[name, "DateOut"] = ( + year + costs.at[carrier, "lifetime"] - 1 + ) df_agg.at[name, "cluster_bus"] = node diff --git a/scripts/retrieve_irena.py b/scripts/retrieve_irena.py deleted file mode 100644 index 04e48db1e..000000000 --- a/scripts/retrieve_irena.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2023 Thomas Gilon (Climact) -# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors -# -# SPDX-License-Identifier: MIT -""" -This rule downloads the existing capacities from `IRENASTAT `_ and extracts it in the ``data/existing_capacities`` sub-directory. - -**Relevant Settings** - -.. code:: yaml - - enable: - retrieve_irena: - -.. seealso:: - Documentation of the configuration file ``config.yaml`` at - :ref:`enable_cf` - -**Outputs** - -- ``data/existing_capacities``: existing capacities for offwind, onwind and solar - -""" - -import logging - -import pandas as pd -from _helpers import configure_logging, set_scenario_config - -logger = logging.getLogger(__name__) - -REGIONS = [ - "Albania", - "Austria", - "Belgium", - "Bosnia and Herzegovina", - "Bulgaria", - "Croatia", - "Czechia", - "Denmark", - "Estonia", - "Finland", - "France", - "Germany", - "Greece", - "Hungary", - "Ireland", - "Italy", - "Latvia", - "Lithuania", - "Luxembourg", - "Montenegro", - # "Netherlands", - "Netherlands (Kingdom of the)", - "North Macedonia", - "Norway", - "Poland", - "Portugal", - "Romania", - "Serbia", - "Slovakia", - "Slovenia", - "Spain", - "Sweden", - "Switzerland", - # "United Kingdom", - "United Kingdom of Great Britain and Northern Ireland (the)", -] - -REGIONS_DICT = { - "Bosnia and Herzegovina": "Bosnia Herzg", - "Netherlands (Kingdom of the)": "Netherlands", - "United Kingdom of Great Britain and Northern Ireland (the)": "UK", -} - -if __name__ == "__main__": - if "snakemake" not in globals(): - from _helpers import mock_snakemake - - snakemake = mock_snakemake("retrieve_irena") - configure_logging(snakemake) - set_scenario_config(snakemake) - - irena_raw = pd.read_csv( - "https://pxweb.irena.org:443/sq/99e64b12-fe03-4a7b-92ea-a22cc3713b92", - skiprows=2, - index_col=[0, 1, 3], - encoding="latin-1", - ) - - var = "Installed electricity capacity (MW)" - irena = irena_raw[var].unstack(level=2).reset_index(level=1).replace(0, "") - - irena = irena[irena.index.isin(REGIONS)] - irena.rename(index=REGIONS_DICT, inplace=True) - - df_offwind = irena[irena.Technology.str.contains("Offshore")].drop( - columns=["Technology"] - ) - df_onwind = irena[irena.Technology.str.contains("Onshore")].drop( - columns=["Technology"] - ) - df_pv = irena[irena.Technology.str.contains("Solar")].drop(columns=["Technology"]) - - df_offwind.to_csv(snakemake.output["offwind"]) - df_onwind.to_csv(snakemake.output["onwind"]) - df_pv.to_csv(snakemake.output["solar"]) From 219847012d381c73c1479bf021cb36d05f797cc0 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 11 Apr 2024 12:56:49 +0200 Subject: [PATCH 10/57] build_bus_regions: fix shapes index for correct alignment in madd cluster_regions: further modularize functions, fix index alignment --- scripts/build_bus_regions.py | 24 +++++++++--------- scripts/cluster_network.py | 48 +++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index d26e6ba6d..d0225797e 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -173,31 +173,31 @@ def voronoi_partition_pts(points, outline): offshore_regions_c = offshore_regions_c.loc[offshore_regions_c.area > 1e-2] offshore_regions.append(offshore_regions_c) - gdf = pd.concat(onshore_regions, ignore_index=True) - gdf.to_file(snakemake.output.regions_onshore) + shapes = pd.concat(onshore_regions, ignore_index=True) + shapes.to_file(snakemake.output.regions_onshore) offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - index = gdf.index.astype(int) + offset + shapes.index = shapes.index.astype(int) + offset n.madd( "Shape", - index, - geometry=gdf.geometry, - idx=index, + shapes.index, + geometry=shapes.geometry, + idx=shapes.name, component="Bus", type="onshore", ) if offshore_regions: - gdf = pd.concat(offshore_regions, ignore_index=True) - gdf.to_file(snakemake.output.regions_offshore) + shapes = pd.concat(offshore_regions, ignore_index=True) + shapes.to_file(snakemake.output.regions_offshore) offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - index = gdf.index.astype(int) + offset + shapes.index = shapes.index.astype(int) + offset n.madd( "Shape", - index, - geometry=gdf.geometry, - idx=index, + shapes.index, + geometry=shapes.geometry, + idx=shapes.name, component="Bus", type="offshore", ) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index a18a20799..9dd0226d8 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -428,7 +428,7 @@ def clustering_for_n_clusters( return clustering -def cluster_regions(n, busmaps, which, input=None, output=None): +def cluster_regions(busmaps, regions): """ Cluster regions based on busmaps and save the results to a file and to the network. @@ -436,36 +436,41 @@ def cluster_regions(n, busmaps, which, input=None, output=None): Parameters: - busmaps (list): A list of busmaps used for clustering. - which (str): The type of regions to cluster. - - input (str, optional): The input file path. Defaults to None. - - output (str, optional): The output file path. Defaults to None. Returns: None """ busmap = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) - - regions = gpd.read_file(getattr(input, which)) regions = regions.reindex(columns=["name", "geometry"]).set_index("name") regions_c = regions.dissolve(busmap) regions_c.index.name = "name" - regions_c = regions_c.reset_index() - regions_c.to_file(getattr(output, which)) + return regions_c.reset_index() + + +def append_bus_shapes(n, shapes, type): + """ + Append shapes to the network. - # remove original regions + Parameters: + n (pypsa.Network): The network to which the shapes will be appended. + shapes (geopandas.GeoDataFrame): The shapes to be appended. + **kwargs: Additional keyword arguments used in `n.madd`. + + Returns: + None + """ remove = n.shapes.query("component == 'Bus' and type == @which").index n.mremove("Shape", remove) - # add new clustered regions offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - index = regions_c.index.astype(int) + offset - type = which.split("_")[1] + shapes.index = shapes.index.astype(int) + offset n.madd( "Shape", - index, - geometry=regions_c.geometry, - idx=index, + shapes.index, + geometry=shapes.geometry, + idx=shapes.name, component="Bus", - type="which", + type=type, ) @@ -574,15 +579,18 @@ def plot_busmap_for_n_clusters(n, n_clusters, solver_name="scip", fn=None): labels = [f" {label} efficiency" for label in ["low", "medium", "high"]] nc.generators["carrier"] = nc.generators.carrier.replace(labels, "", regex=True) - nc.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - nc.export_to_netcdf(snakemake.output.network) for attr in ( "busmap", "linemap", ): # also available: linemap_positive, linemap_negative getattr(clustering, attr).to_csv(snakemake.output[attr]) + nc.shapes = n.shapes.copy() for which in ["regions_onshore", "regions_offshore"]: - cluster_regions( - nc, (clustering.busmap,), which, snakemake.input, snakemake.output - ) + regions = gpd.read_file(snakemake.input[which]) + clustered_regions = cluster_regions((clustering.busmap,), regions) + append_bus_shapes(nc, clustered_regions, type=which.split("_")[1]) + clustered_regions.to_file(snakemake.output[which]) + + nc.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + nc.export_to_netcdf(snakemake.output.network) From 6d9728aed6218950373a496bdf6dbdc524bfb3be Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 11 Apr 2024 13:02:28 +0200 Subject: [PATCH 11/57] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d42b149f8..d7e154b75 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -181,6 +181,8 @@ Upcoming Release * Fix custom busmap read in `cluster_network`. +* Data on existing renewable capacities is now consistently taken from powerplantmatching (instead of being retrieved separately); the dataset has also been updated to include 2023 values. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== From 6965ce31ccca19b88ded25b0102973b87b333b9f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:05:35 +0000 Subject: [PATCH 12/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d7e154b75..b37d3c4dd 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -181,7 +181,7 @@ Upcoming Release * Fix custom busmap read in `cluster_network`. -* Data on existing renewable capacities is now consistently taken from powerplantmatching (instead of being retrieved separately); the dataset has also been updated to include 2023 values. +* Data on existing renewable capacities is now consistently taken from powerplantmatching (instead of being retrieved separately); the dataset has also been updated to include 2023 values. PyPSA-Eur 0.10.0 (19th February 2024) ===================================== From f1b33992f7adc8941e668d61eb30dbf4e46d233d Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 11 Apr 2024 13:27:52 +0200 Subject: [PATCH 13/57] move append_bus_shapes to build_bus_regions, apply it where possible --- scripts/build_bus_regions.py | 58 ++++++++++++++++++++++-------------- scripts/cluster_network.py | 30 ++----------------- scripts/simplify_network.py | 15 +++++++--- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index d0225797e..05a7729ef 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -109,6 +109,34 @@ def voronoi_partition_pts(points, outline): return polygons +def append_bus_shapes(n, shapes, type): + """ + Append shapes to the network. If shapes with the same component and type + already exist, they will be removed. + + Parameters: + n (pypsa.Network): The network to which the shapes will be appended. + shapes (geopandas.GeoDataFrame): The shapes to be appended. + **kwargs: Additional keyword arguments used in `n.madd`. + + Returns: + None + """ + remove = n.shapes.query("component == 'Bus' and type == @type").index + n.mremove("Shape", remove) + + offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 + shapes = shapes.rename(lambda x: int(x) + offset) + n.madd( + "Shape", + shapes.index, + geometry=shapes.geometry, + idx=shapes.name, + component="Bus", + type=type, + ) + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -119,7 +147,8 @@ def voronoi_partition_pts(points, outline): countries = snakemake.params.countries - n = pypsa.Network(snakemake.input.base_network) + base_network = snakemake.input.base_network + n = pypsa.Network(base_network) country_shapes = gpd.read_file(snakemake.input.country_shapes).set_index("name")[ "geometry" @@ -175,32 +204,15 @@ def voronoi_partition_pts(points, outline): shapes = pd.concat(onshore_regions, ignore_index=True) shapes.to_file(snakemake.output.regions_onshore) - - offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - shapes.index = shapes.index.astype(int) + offset - n.madd( - "Shape", - shapes.index, - geometry=shapes.geometry, - idx=shapes.name, - component="Bus", - type="onshore", - ) + append_bus_shapes(n, shapes, "onshore") if offshore_regions: shapes = pd.concat(offshore_regions, ignore_index=True) shapes.to_file(snakemake.output.regions_offshore) - - offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - shapes.index = shapes.index.astype(int) + offset - n.madd( - "Shape", - shapes.index, - geometry=shapes.geometry, - idx=shapes.name, - component="Bus", - type="offshore", - ) + append_bus_shapes(n, shapes, "offshore") else: offshore_shapes.to_frame().to_file(snakemake.output.regions_offshore) + + # save network with shapes + n.export_to_netcdf(base_network) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 9dd0226d8..f58e5f8ba 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -135,6 +135,7 @@ import seaborn as sns from _helpers import configure_logging, set_scenario_config, update_p_nom_max from add_electricity import load_costs +from build_bus_regions import append_bus_shapes from packaging.version import Version, parse from pypsa.clustering.spatial import ( busmap_by_greedy_modularity, @@ -447,33 +448,6 @@ def cluster_regions(busmaps, regions): return regions_c.reset_index() -def append_bus_shapes(n, shapes, type): - """ - Append shapes to the network. - - Parameters: - n (pypsa.Network): The network to which the shapes will be appended. - shapes (geopandas.GeoDataFrame): The shapes to be appended. - **kwargs: Additional keyword arguments used in `n.madd`. - - Returns: - None - """ - remove = n.shapes.query("component == 'Bus' and type == @which").index - n.mremove("Shape", remove) - - offset = n.shapes.index.astype(int).max() + 1 if not n.shapes.empty else 0 - shapes.index = shapes.index.astype(int) + offset - n.madd( - "Shape", - shapes.index, - geometry=shapes.geometry, - idx=shapes.name, - component="Bus", - type=type, - ) - - def plot_busmap_for_n_clusters(n, n_clusters, solver_name="scip", fn=None): busmap = busmap_for_n_clusters(n, n_clusters, solver_name) cs = busmap.unique() @@ -589,8 +563,8 @@ def plot_busmap_for_n_clusters(n, n_clusters, solver_name="scip", fn=None): for which in ["regions_onshore", "regions_offshore"]: regions = gpd.read_file(snakemake.input[which]) clustered_regions = cluster_regions((clustering.busmap,), regions) - append_bus_shapes(nc, clustered_regions, type=which.split("_")[1]) clustered_regions.to_file(snakemake.output[which]) + append_bus_shapes(nc, clustered_regions, type=which.split("_")[1]) nc.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) nc.export_to_netcdf(snakemake.output.network) diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 7b8710a05..f129945ca 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -88,12 +88,14 @@ import logging from functools import reduce +import geopandas as gpd import numpy as np import pandas as pd import pypsa import scipy as sp from _helpers import configure_logging, set_scenario_config, update_p_nom_max from add_electricity import load_costs +from build_bus_regions import append_bus_shapes from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( aggregateoneport, @@ -610,6 +612,7 @@ def cluster( n.lines.drop(remove, axis=1, errors="ignore", inplace=True) if snakemake.wildcards.simpl: + shapes = n.shapes n, cluster_map = cluster( n, int(snakemake.wildcards.simpl), @@ -619,15 +622,19 @@ def cluster( params.simplify_network["feature"], params.aggregation_strategies, ) + n.shapes = shapes busmaps.append(cluster_map) update_p_nom_max(n) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output.network) - busmap_s = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0]) busmap_s.to_csv(snakemake.output.busmap) for which in ["regions_onshore", "regions_offshore"]: - cluster_regions(n, busmaps, which, snakemake.input, snakemake.output) + regions = gpd.read_file(snakemake.input[which]) + clustered_regions = cluster_regions(busmaps, regions) + clustered_regions.to_file(snakemake.output[which]) + append_bus_shapes(n, clustered_regions, type=which.split("_")[1]) + + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output.network) From 7c30a2835fbe0f8435ca38b38c45a155bc3554c8 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 14 Mar 2024 13:44:15 +0100 Subject: [PATCH 14/57] group existing capacities to the earlier grouping_year for consistency with optimized capacities --- config/config.default.yaml | 2 +- doc/release_notes.rst | 1 + scripts/add_existing_baseyear.py | 20 ++++++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index d438c51f7..0d40bafc9 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -361,7 +361,7 @@ solar_thermal: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] - grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 + grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020] # heat grouping years >= baseyear will be ignored threshold_capacity: 10 default_heating_lifetime: 20 conventional_carriers: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d42b149f8..cdad3fa0f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,6 +9,7 @@ Release Notes Upcoming Release ================ +* Group existing capacities to the earlier grouping_year for consistency with optimized capacities * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential. diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f445efe34..75d83e385 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -190,8 +190,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas df_agg.drop(phased_out, inplace=True) df_agg["grouping_year"] = np.take( - grouping_years, np.digitize(df_agg.DateIn, grouping_years, right=True) - ) + grouping_years[::-1], + np.digitize(df_agg.DateIn, grouping_years[::-1])) # calculate (adjusted) remaining lifetime before phase-out (+1 because assuming # phase out date at the end of the year) @@ -444,12 +444,16 @@ def add_heating_capacities_installed_before_baseyear( else: efficiency = costs.at[costs_name, "efficiency"] - for i, grouping_year in enumerate(grouping_years): - if int(grouping_year) + default_lifetime <= int(baseyear): - continue - - # installation is assumed to be linear for the past default_lifetime years - ratio = (int(grouping_year) - int(grouping_years[i - 1])) / default_lifetime + valid_grouping_years = pd.Series( + [int(gy) for gy in grouping_years if + int(baseyear) - default_lifetime <= int(gy) < int(baseyear)]) + # Installation is assumed to be linear for the past + _intervals = pd.concat( + [valid_grouping_years[1:], pd.Series(baseyear)], + ignore_index=True) - valid_grouping_years + ratios = _intervals / _intervals.sum() + + for ratio, grouping_year in zip(ratios, valid_grouping_years): n.madd( "Link", From 700b0e912a31c14e67db0ff4cc504590d7242448 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 14 Mar 2024 16:13:40 +0100 Subject: [PATCH 15/57] add more grouping years to avoid errors --- config/config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 0d40bafc9..42132f226 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -360,7 +360,7 @@ solar_thermal: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: - grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] + grouping_years_power: [1895, 1920, 1950, 1955, 1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020] # heat grouping years >= baseyear will be ignored threshold_capacity: 10 default_heating_lifetime: 20 From 3a0e3563821b4f3cded77c28ce57f23223751e4c Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 08:54:15 +0200 Subject: [PATCH 16/57] add check if minimum grouping year< min build year --- scripts/add_existing_baseyear.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 75d83e385..01870a26b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -189,6 +189,15 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas phased_out = df_agg[df_agg["DateOut"] < baseyear].index df_agg.drop(phased_out, inplace=True) + older_assets = (df_agg.DateIn Date: Thu, 11 Apr 2024 09:26:30 +0200 Subject: [PATCH 17/57] adjust valid_grouping_years change sign, split up conditions for better readibility --- scripts/add_existing_baseyear.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 01870a26b..75375565b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -454,14 +454,22 @@ def add_heating_capacities_installed_before_baseyear( efficiency = costs.at[costs_name, "efficiency"] valid_grouping_years = pd.Series( - [int(gy) for gy in grouping_years if - int(baseyear) - default_lifetime <= int(gy) < int(baseyear)]) + [ + int(grouping_year) + for grouping_year in grouping_years + if int(grouping_year) + default_lifetime > int(baseyear) + and int(grouping_year) < int(baseyear) + ] + ) # Installation is assumed to be linear for the past - _intervals = pd.concat( - [valid_grouping_years[1:], pd.Series(baseyear)], - ignore_index=True) - valid_grouping_years + _intervals = ( + pd.concat( + [valid_grouping_years[1:], pd.Series(baseyear)], ignore_index=True + ) + - valid_grouping_years + ) ratios = _intervals / _intervals.sum() - + for ratio, grouping_year in zip(ratios, valid_grouping_years): n.madd( From 7773f7c84ebbbbdd8dd9bd5e3b2f8e1ed6edbc7d Mon Sep 17 00:00:00 2001 From: Micha Date: Thu, 11 Apr 2024 09:37:36 +0200 Subject: [PATCH 18/57] Add release note for serendipitous bugfix --- doc/release_notes.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cdad3fa0f..079648d50 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,7 +9,9 @@ Release Notes Upcoming Release ================ -* Group existing capacities to the earlier grouping_year for consistency with optimized capacities +* Group existing capacities to the earlier grouping_year for consistency with optimized capacities. + +* bugfix: installed heating capacities were 5% lower than existing heating capacities * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential. From e9c9d08107967119539b3fa7202b3f670868d092 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:52:17 +0200 Subject: [PATCH 19/57] rename _intervals to _years, shorten definition --- scripts/add_existing_baseyear.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 75375565b..d70afb485 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -461,14 +461,12 @@ def add_heating_capacities_installed_before_baseyear( and int(grouping_year) < int(baseyear) ] ) + + # get number of years of each interval + _years = (valid_grouping_years.diff().shift(-1) + .fillna(baseyear-valid_grouping_years.iloc[-1])) # Installation is assumed to be linear for the past - _intervals = ( - pd.concat( - [valid_grouping_years[1:], pd.Series(baseyear)], ignore_index=True - ) - - valid_grouping_years - ) - ratios = _intervals / _intervals.sum() + ratios = _years / _years.sum() for ratio, grouping_year in zip(ratios, valid_grouping_years): From a72388b989d1d667ec7e44d66f6c3b494b46d000 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:54:33 +0200 Subject: [PATCH 20/57] define active assets consitently --- scripts/add_brownfield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 7e49031b7..16b4e0870 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -40,8 +40,8 @@ def add_brownfield(n, n_p, year): # CO2 or global EU values since these are already in n n_p.mremove(c.name, c.df.index[c.df.lifetime == np.inf]) - # remove assets whose build_year + lifetime < year - n_p.mremove(c.name, c.df.index[c.df.build_year + c.df.lifetime < year]) + # remove assets whose build_year + lifetime <= year + n_p.mremove(c.name, c.df.index[c.df.build_year + c.df.lifetime <= year]) # remove assets if their optimized nominal capacity is lower than a threshold # since CHP heat Link is proportional to CHP electric Link, make sure threshold is compatible From f8b33e8eb28982b424e3637c96fa53f44bb2b8cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 12:21:43 +0000 Subject: [PATCH 21/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d70afb485..8a8a78a4a 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -189,18 +189,20 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas phased_out = df_agg[df_agg["DateOut"] < baseyear].index df_agg.drop(phased_out, inplace=True) - older_assets = (df_agg.DateIn Date: Thu, 11 Apr 2024 15:37:54 +0200 Subject: [PATCH 22/57] Activate Sphinx automodules, add mock imports for missing pypsa-eur modules in environment used to build documentation --- doc/conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index afa01d3ab..c9f0b2567 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -34,10 +34,10 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - #'sphinx.ext.autodoc', + 'sphinx.ext.autodoc', #'sphinx.ext.autosummary', "myst_parser", - "sphinx.ext.autosectionlabel", + # "sphinx.ext.autosectionlabel", "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx.ext.mathjax", @@ -50,6 +50,8 @@ "sphinx.ext.imgconverter", # for SVG conversion ] +autodoc_mock_imports = ["atlite", "snakemake", "pycountry", "rioxarray", "country_converter", "tabula", "memory_profiler", "powerplantmatching", "rasterio", "dask.distributed"] + autodoc_default_flags = ["members"] autosummary_generate = True From 1c321b72cf57b0d24307cff96e3967d0bbcda2a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 14:53:09 +0000 Subject: [PATCH 23/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/conf.py | 15 +++++++++++++-- doc/contributing.rst | 1 - 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index c9f0b2567..739c76635 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -34,7 +34,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', + "sphinx.ext.autodoc", #'sphinx.ext.autosummary', "myst_parser", # "sphinx.ext.autosectionlabel", @@ -50,7 +50,18 @@ "sphinx.ext.imgconverter", # for SVG conversion ] -autodoc_mock_imports = ["atlite", "snakemake", "pycountry", "rioxarray", "country_converter", "tabula", "memory_profiler", "powerplantmatching", "rasterio", "dask.distributed"] +autodoc_mock_imports = [ + "atlite", + "snakemake", + "pycountry", + "rioxarray", + "country_converter", + "tabula", + "memory_profiler", + "powerplantmatching", + "rasterio", + "dask.distributed", +] autodoc_default_flags = ["members"] autosummary_generate = True diff --git a/doc/contributing.rst b/doc/contributing.rst index 52dc8280b..22dfb4474 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -47,4 +47,3 @@ We strive to keep documentation useful and up to date for all PyPSA users. If yo #. Compile your changes by running the following command in your terminal in the ``doc`` folder: ``make html`` You may encounter some warnings, but end up with a message such as ``build succeeded, XX warnings.``. html files to review your changes can then be found under ``doc/_build/html``. #. Contribute your documentation in a pull request (`here is a guide `_). - From cb7061d6030d73e3d01b7f0c192c65263977d7b5 Mon Sep 17 00:00:00 2001 From: chrstphtrs Date: Thu, 11 Apr 2024 15:55:11 +0200 Subject: [PATCH 24/57] Change all documentation links to be anonymous by adding double underscore suffix to avoid automatic target creation (which is triggered by single underscore suffix) --- doc/configuration.rst | 12 +- doc/costs.rst | 4 +- doc/foresight.rst | 4 +- doc/index.rst | 44 +++--- doc/installation.rst | 26 ++-- doc/introduction.rst | 14 +- doc/licenses.rst | 12 +- doc/limitations.rst | 2 +- doc/preparation.rst | 8 +- doc/release_notes.rst | 298 ++++++++++++++++++------------------- doc/retrieve.rst | 20 +-- doc/sector.rst | 11 +- doc/simplification.rst | 4 +- doc/spatial_resolution.rst | 14 +- doc/supply_demand.rst | 184 +++++++++++------------ doc/support.rst | 10 +- doc/tutorial.rst | 2 +- doc/validation.rst | 2 +- doc/wildcards.rst | 2 +- 19 files changed, 340 insertions(+), 333 deletions(-) diff --git a/doc/configuration.rst b/doc/configuration.rst index dae913801..d531ee66a 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -90,9 +90,9 @@ For each wildcard, a **list of values** is provided. The rule ``results/networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc`` for **all combinations** of the provided wildcard values as defined by Python's `itertools.product(...) -`_ function +`__ function that snakemake's `expand(...) function -`_ +`__ uses. An exemplary dependency graph (starting from the simplification rules) then looks like this: @@ -129,7 +129,7 @@ An exemplary dependency graph (starting from the simplification rules) then look ``snapshots`` ============= -Specifies the temporal range to build an energy system model for as arguments to `pandas.date_range `_ +Specifies the temporal range to build an energy system model for as arguments to `pandas.date_range `__ .. literalinclude:: ../config/config.default.yaml :language: yaml @@ -197,7 +197,7 @@ Switches for some rules and optional features. ``atlite`` ========== -Define and specify the ``atlite.Cutout`` used for calculating renewable potentials and time-series. All options except for ``features`` are directly used as `cutout parameters `_. +Define and specify the ``atlite.Cutout`` used for calculating renewable potentials and time-series. All options except for ``features`` are directly used as `cutout parameters `__. .. literalinclude:: ../config/config.default.yaml :language: yaml @@ -427,7 +427,7 @@ overwrite the existing values. :widths: 22,7,22,33 :file: configtables/biomass.csv -The list of available biomass is given by the category in `ENSPRESO_BIOMASS `_, namely: +The list of available biomass is given by the category in `ENSPRESO_BIOMASS `__, namely: - Agricultural waste - Manure solid, liquid @@ -564,7 +564,7 @@ The list of available biomass is given by the category in `ENSPRESO_BIOMASS `_ and then +`PyPSA/technology-data `__ and then saved to a file ``resources/costs_{year}.csv``. The ``config/config.yaml`` provides options to choose a reference year and use a specific version of the repository. @@ -30,7 +30,7 @@ years compiled from various sources, namely for - carbon-dioxide intensity. Many values are taken from a database published by the Danish Energy Agency (`DEA -`_). +`__). The given overnight capital costs are annualised to net present costs diff --git a/doc/foresight.rst b/doc/foresight.rst index c749c84c2..400f67cee 100644 --- a/doc/foresight.rst +++ b/doc/foresight.rst @@ -166,13 +166,13 @@ Options The total carbon budget for the entire transition path can be indicated in the `sector_opts -`_ +`__ in ``config/config.yaml``. The carbon budget can be split among the ``planning_horizons`` following an exponential or beta decay. E.g. ``'cb40ex0'`` splits a carbon budget equal to 40 Gt :math:`_{CO_2}` following an exponential decay whose initial linear growth rate r is zero. They can also follow some user-specified path, if defined `here -`_. +`__. The paper `Speed of technological transformations required in Europe to achieve different climate goals (2022) `__ defines CO_2 budgets corresponding to global temperature increases (1.5C – 2C) diff --git a/doc/index.rst b/doc/index.rst index acff820b4..07ee28d10 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -81,16 +81,16 @@ them: .. note:: You can find showcases of the model's capabilities in the Supplementary Materials of the Joule paper `The potential role of a hydrogen network in Europe - `_, the Supplementary Materials of another `paper in Joule with a + `__, the Supplementary Materials of another `paper in Joule with a description of the industry sector - `_, or in `a 2021 presentation - at EMP-E `_. + `__, or in `a 2021 presentation + at EMP-E `__. The sector-coupled extension of PyPSA-Eur was initially described in the paper `Synergies of sector coupling and transmission reinforcement in a cost-optimised, highly renewable European energy system - `_ (2018) but it differs by being based on the + `__ (2018) but it differs by being based on the higher resolution electricity transmission model `PyPSA-Eur - `_ rather than a one-node-per-country model, + `__ rather than a one-node-per-country model, and by including biomass, industry, industrial feedstocks, aviation, shipping, better carbon management, carbon capture and usage/sequestration, and gas networks. @@ -99,8 +99,8 @@ About ===== PyPSA-Eur is designed to be imported into the open energy system modelling -framework `PyPSA `_ for which `documentation -`_ is available as well. However, since the +framework `PyPSA `__ for which `documentation +`__ is available as well. However, since the workflow is modular, it should be easy to adapt the data workflow to other modelling frameworks. @@ -114,22 +114,22 @@ of the individual parts. PyPSA-Eur is under active development and has several :doc:`limitations` which you should understand before using the model. The Github repository - `issues `_ collect known + `issues `__ collect known topics we are working on. Please feel free to help or make suggestions. This project is currently maintained by the `Department of Digital -Transformation in Energy Systems `_ at the -`Technische Universität Berlin `_. Previous versions were -developed within the `IAI `_ at the `Karlsruhe Institute -of Technology (KIT) `_ which was funded by -the `Helmholtz Association `_, and by the +Transformation in Energy Systems `__ at the +`Technische Universität Berlin `__. Previous versions were +developed within the `IAI `__ at the `Karlsruhe Institute +of Technology (KIT) `__ which was funded by +the `Helmholtz Association `__, and by the `Renewable Energy Group -`_ -at `FIAS `_ to carry out simulations for the -`CoNDyNet project `_, financed by the `German Federal -Ministry for Education and Research (BMBF) `_ +`__ +at `FIAS `__ to carry out simulations for the +`CoNDyNet project `__, financed by the `German Federal +Ministry for Education and Research (BMBF) `__ as part of the `Stromnetze Research Initiative -`_. +`__. Workflow @@ -153,10 +153,10 @@ to reading this documentation. - Documentation of `PyPSA `__, the package for modelling energy systems which PyPSA-Eur uses under the hood. -- Course on `Energy Systems `_ given at - Technical University of Berlin by `Prof. Dr. Tom Brown `_. -- Course on `Data Science for Energy System Modelling `_ - given at Technical University of Berlin by `Dr. Fabian Neumann `_. +- Course on `Energy Systems `__ given at + Technical University of Berlin by `Prof. Dr. Tom Brown `__. +- Course on `Data Science for Energy System Modelling `__ + given at Technical University of Berlin by `Dr. Fabian Neumann `__. Citing PyPSA-Eur diff --git a/doc/installation.rst b/doc/installation.rst index fbabfd158..dd5c912e6 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -15,7 +15,7 @@ directory in which the commands following the ``%`` should be entered. Clone the Repository ==================== -First of all, clone the `PyPSA-Eur repository `_ using the version control system ``git`` in the command line. +First of all, clone the `PyPSA-Eur repository `__ using the version control system ``git`` in the command line. .. code:: bash @@ -30,11 +30,11 @@ Install Python Dependencies =============================== PyPSA-Eur relies on a set of other Python packages to function. -We recommend using the package manager `mamba `_ to install them and manage your environments. -For instructions for your operating system follow the ``mamba`` `installation guide `_. +We recommend using the package manager `mamba `__ to install them and manage your environments. +For instructions for your operating system follow the ``mamba`` `installation guide `__. You can also use ``conda`` equivalently. -The package requirements are curated in the `envs/environment.yaml `_ file. +The package requirements are curated in the `envs/environment.yaml `__ file. The environment can be installed and activated using .. code:: bash @@ -59,16 +59,16 @@ Install a Solver PyPSA passes the PyPSA-Eur network model to an external solver for performing the optimisation. PyPSA is known to work with the free software -- `HiGHS `_ -- `Cbc `_ -- `GLPK `_ (`WinGLKP `_) -- `Ipopt `_ +- `HiGHS `__ +- `Cbc `__ +- `GLPK `__ (`WinGLKP `__) +- `Ipopt `__ and the non-free, commercial software (for some of which free academic licenses are available) -- `Gurobi `_ -- `CPLEX `_ -- `FICO Xpress Solver `_ +- `Gurobi `__ +- `CPLEX `__ +- `FICO Xpress Solver `__ For installation instructions of these solvers for your operating system, follow the links above. Commercial solvers such as Gurobi and CPLEX currently significantly outperform open-source solvers for large-scale problems, and @@ -76,7 +76,7 @@ it might be the case that you can only retrieve solutions by using a commercial Nevertheless, you can still use open-source solvers for smaller problems. .. seealso:: - `Instructions how to install a solver in the documentation of PyPSA `_ + `Instructions how to install a solver in the documentation of PyPSA `__ .. note:: The rules :mod:`cluster_network` and :mod:`simplify_network` solve a mixed-integer quadratic optimisation problem for clustering. @@ -88,7 +88,7 @@ Nevertheless, you can still use open-source solvers for smaller problems. mamba activate pypsa-eur mamba install -c gurobi gurobi - Additionally, you need to setup your `Gurobi license `_. + Additionally, you need to setup your `Gurobi license `__. .. _defaultconfig: diff --git a/doc/introduction.rst b/doc/introduction.rst index d271391c8..7cfa0e43f 100644 --- a/doc/introduction.rst +++ b/doc/introduction.rst @@ -14,7 +14,7 @@ .. note:: - Find the introductory slides `here `_. + Find the introductory slides `here `__. .. warning:: The video only introduces the electricity-only part of PyPSA-Eur. @@ -23,7 +23,7 @@ Workflow ========= The generation of the model is controlled by the open workflow management system -`Snakemake `_. In a nutshell, the ``Snakefile`` +`Snakemake `__. In a nutshell, the ``Snakefile`` declares for each script in the ``scripts`` directory a rule which describes which files the scripts consume and produce (their corresponding input and output files). The ``snakemake`` tool then runs the scripts in the correct order @@ -54,9 +54,9 @@ preceding rules which another rule takes as input data. For the use of ``snakemake``, it makes sense to familiarize yourself quickly with the `basic tutorial -`_ and then +`__ and then read carefully through the documentation of the `command line interface -`_, noting the +`__, noting the arguments ``-j``, ``-c``, ``-f``, ``-F``, ``-n``, ``-r``, ``--dag`` and ``-t`` in particular. @@ -64,10 +64,10 @@ Scenarios, Configuration and Modification ========================================= It is easy to run PyPSA-Eur for multiple scenarios using the `wildcards feature -`_ +`__ of ``snakemake``. Wildcards allow to generalise a rule to produce all files that follow a `regular expression -`_ pattern, which defines +`__ pattern, which defines a particular scenario. One can think of a wildcard as a parameter that shows up in the input/output file names and thereby determines which rules to run, what data to retrieve and what files to produce. Details are explained in @@ -97,5 +97,5 @@ System Requirements Building the model with the scripts in this repository runs on a regular computer. But optimising for investment and operation decisions across many scenarios requires a strong interior-point solver -like `Gurobi `_ or `CPLEX `_ with more memory. +like `Gurobi `__ or `CPLEX `__ with more memory. Open-source solvers like `HiGHS ` can also be used for smaller problems. diff --git a/doc/licenses.rst b/doc/licenses.rst index 74640ea55..af531660e 100644 --- a/doc/licenses.rst +++ b/doc/licenses.rst @@ -10,12 +10,12 @@ Licenses PyPSA-Eur is released under multiple licenses: -* All original source code is licensed as free software under `MIT `_. -* The documentation is licensed under `CC-BY-4.0 `_. -* Configuration files are mostly licensed under `CC0-1.0 `_. -* Data files are licensed under `CC-BY-4.0 `_. +* All original source code is licensed as free software under `MIT `__. +* The documentation is licensed under `CC-BY-4.0 `__. +* Configuration files are mostly licensed under `CC0-1.0 `__. +* Data files are licensed under `CC-BY-4.0 `__. -See the individual files and the `dep5 <.reuse/dep5>`_ file for license details. +See the individual files and the `dep5 <.reuse/dep5>`__ file for license details. Additionally, different licenses and terms of use also apply to the various input data for both electricity-only and sector-coupled modelling exercises, @@ -26,7 +26,7 @@ Electricity Systems Databundle .. note:: More details are included in `the description of the - data bundles on zenodo `_. + data bundles on zenodo `__. .. csv-table:: :header-rows: 1 diff --git a/doc/limitations.rst b/doc/limitations.rst index 2aa8ecfe2..aeec5da68 100644 --- a/doc/limitations.rst +++ b/doc/limitations.rst @@ -19,7 +19,7 @@ improving the approximations. This list of limitations is incomplete and will be added to over time. .. seealso:: - See also the `GitHub repository issues `_. + See also the `GitHub repository issues `__. - **Electricity transmission network topology:** The grid data is based on a map of the ENTSO-E area that is known diff --git a/doc/preparation.rst b/doc/preparation.rst index bb55ba6bf..f608e031a 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -15,18 +15,18 @@ Instead we provide separate data bundles which can be obtained using the ``retrieve*`` rules (:ref:`data`). Having downloaded the necessary data, -- :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 `_ areas. -- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 `_ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``. +- :mod:`build_shapes` generates GeoJSON files with shapes of the countries, exclusive economic zones and `NUTS3 `__ areas. +- :mod:`build_cutout` prepares smaller weather data portions from `ERA5 `__ for cutout ``europe-2013-era5`` and SARAH for cutout ``europe-2013-sarah``. With these and the externally extracted ENTSO-E online map topology (``data/entsoegridkit``), it can build a base PyPSA network with the following rules: - :mod:`base_network` builds and stores the base network with all buses, HVAC lines and HVDC links, while -- :mod:`build_bus_regions` determines `Voronoi cells `_ for all substations. +- :mod:`build_bus_regions` determines `Voronoi cells `__ for all substations. Then the process continues by calculating conventional power plant capacities, potentials, and per-unit availability time series for variable renewable energy carriers and hydro power plants with the following rules: -- :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching `_ allocating these to the closest substation for each powerplant, +- :mod:`build_powerplants` for today's thermal power plant capacities using `powerplantmatching `__ allocating these to the closest substation for each powerplant, - :mod:`build_natura_raster` for rasterising NATURA2000 natural protection areas, - :mod:`build_ship_raster` for building shipping traffic density, - :mod:`build_renewable_profiles` for the hourly capacity factors and installation potentials constrained by land-use in each substation's Voronoi cell for PV, onshore and offshore wind, and diff --git a/doc/release_notes.rst b/doc/release_notes.rst index c50999436..74ca724b9 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -51,20 +51,20 @@ Upcoming Release - Include time series of Swiss number of passenger vehicles from the `Swiss Federal Statistical Office - `_. + `__. - Updated hydro-electricity generation and capacity data from EIA. - The easiest way to sweep over multiple weather years is to use the new scenario management. An example for the necessary `create_scenarios.py` script can be found in this `Github gist - `_. + `__. * Removed rule ``copy_config``. Instead, a config file is created for each network output of the ``solve_*`` rules, with the same content as ``n.meta``. * Added new HVDC transmission projects from `TYNDP 2024 draft projects - `_. + `__. * Upgrade to Snakemake v8.5+. This version is the new minimum version required. To upgrade an existing environment, run ``conda install -c bioconda @@ -80,7 +80,7 @@ Upcoming Release * The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`. * The latest `Swiss energy totals - `_ + `__ have been updated to the 2023 version. * The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after @@ -269,10 +269,10 @@ PyPSA-Eur 0.10.0 (19th February 2024) * Default settings for recycling rates and primary product shares of high-value chemicals have been set in accordance with the values used in `Neumann et al. - (2023) `_ linearly interpolated + (2023) `__ linearly interpolated between 2020 and 2050. The recycling rates are based on data from `Agora Energiewende (2021) - `_. + `__. * Air-sourced heat pumps can now also be built in rural areas. Previously, only ground-sourced heat pumps were considered for this category @@ -390,7 +390,7 @@ PyPSA-Eur 0.10.0 (19th February 2024) (https://github.com/PyPSA/pypsa-eur/pull/861). Special thanks for this release to Koen van Greevenbroek (`@koen-vg -`_) for various new features, bugfixes and taking +`__) for various new features, bugfixes and taking care of deprecations. @@ -427,18 +427,18 @@ PyPSA-Eur 0.9.0 (5th January 2024) * Add locations, capacities and costs of existing gas storage using Global Energy Monitor's `Europe Gas Tracker - `_ + `__ (https://github.com/PyPSA/pypsa-eur/pull/835). * Add option to use `LUISA Base Map - `_ 50m land + `__ 50m land coverage dataset for land eligibility analysis in :mod:`build_renewable_profiles`. Settings are analogous to the CORINE dataset but with the key ``luisa:`` in the configuration file. To leverage the dataset's full advantages, set the excluder resolution to 50m (``excluder_resolution: 50``). For land category codes, see `Annex 1 of the technical documentation - `_ + `__ (https://github.com/PyPSA/pypsa-eur/pull/842). * Add option to capture CO2 contained in biogas when upgrading (``sector: @@ -681,7 +681,7 @@ PyPSA-Eur 0.8.1 (27th July 2023) * Add option to consider dynamic line rating based on wind speeds and temperature according to `Glaum and Hofmann (2022) - `_. See configuration section ``lines: + `__. See configuration section ``lines: dynamic_line_rating:`` for more details. (https://github.com/PyPSA/pypsa-eur/pull/675) * Add option to include a piecewise linear approximation of transmission losses, @@ -700,7 +700,7 @@ PyPSA-Eur 0.8.1 (27th July 2023) * A ``param:`` section in the snakemake rule definitions was added to track changed settings in ``config.yaml``. The goal is to automatically re-execute rules where parameters have changed. See `Non-file parameters for rules - `_ + `__ in the snakemake documentation. (https://github.com/PyPSA/pypsa-eur/pull/663) * A new function named ``sanitize_carrier`` ensures that all unique carrier @@ -809,7 +809,7 @@ PyPSA-Eur 0.8.0 (18th March 2023) * The :mod:`solve_network` script now uses the ``linopy`` backend of PyPSA and is applied for both electricity-only and sector-coupled models. This requires an adjustment of custom ``extra_functionality``. - See the `migration guide `_ in the PyPSA documentation. + See the `migration guide `__ in the PyPSA documentation. * The configuration file ``config.default.yaml`` now also includes settings for sector-coupled models, which will be ignored when the user runs @@ -914,7 +914,7 @@ PyPSA-Eur 0.7.0 (16th February 2023) inclusive:`` to address the upstream deprecation with ``pandas=1.4``. The previous setting ``None`` is no longer supported and replaced by ``both``, see the `pandas documentation - `_. + `__. Minimum version is now ``pandas>=1.4``. * The configuration setting ``summary_dir`` was removed. @@ -968,7 +968,7 @@ PyPSA-Eur 0.6.1 (20th September 2022) * Individual commits are now tested against pre-commit hooks. This includes black style formatting, sorting of package imports, Snakefile formatting and others. Installation instructions can for the pre-commit can be found `here - `_. + `__. * Pre-commit CI is now part of the repository's CI. @@ -991,7 +991,7 @@ PyPSA-Eur 0.6.0 (10th September 2022) * Functionality to consider shipping routes when calculating the available area for offshore technologies were added. Data for the shipping density comes from the `Global Shipping Traffic Density dataset - `_. + `__. * When transforming all transmission lines to a unified voltage level of 380kV, the workflow now preserves the transmission capacity rather than electrical @@ -1028,7 +1028,7 @@ PyPSA-Eur 0.5.0 (27th July 2022) ``from_opsd`` to ``True``. * Add operational reserve margin constraint analogous to `GenX implementation - `_. Can be activated + `__. Can be activated with config setting ``electricity: operational_reserve:``. * Implement country-specific Energy Availability Factors (EAFs) for nuclear @@ -1054,12 +1054,12 @@ PyPSA-Eur 0.5.0 (27th July 2022) * Techno-economic parameters of technologies (e.g. costs and efficiencies) will now be retrieved from a separate repository `PyPSA/technology-data - `_ that collects assumptions from a + `__ that collects assumptions from a variety of sources. It is activated by default with ``enable: retrieve_cost_data: true`` and controlled with ``costs: year:`` and ``costs: version:``. The location of this data changed from ``data/costs.csv`` to ``resources/costs.csv`` [`#184 - `_]. + `__]. * A new section ``conventional`` was added to the config file. This section contains configurations for conventional carriers. @@ -1074,18 +1074,18 @@ PyPSA-Eur 0.5.0 (27th July 2022) * Add an efficiency factor of 88.55% to offshore wind capacity factors as a proxy for wake losses. More rigorous modelling is `planned - `_ [`#277 - `_]. + `__ [`#277 + `__]. * Following discussion in `#285 - `_ we have disabled the + `__ we have disabled the correction factor for solar PV capacity factors by default while satellite data is used. A correction factor of 0.854337 is recommended if reanalysis data like ERA5 is used. * The default deployment density of AC- and DC-connected offshore wind capacity is reduced from 3 MW/sqkm to a more conservative estimate of 2 MW/sqkm [`#280 - `_]. + `__]. * The inclusion of renewable carriers is now specified in the config entry ``renewable_carriers``. Before this was done by commenting/uncommenting @@ -1114,12 +1114,12 @@ PyPSA-Eur 0.5.0 (27th July 2022) * Resource definitions for memory usage now follow `Snakemake standard resource definition - `_ + `__ ``mem_mb`` rather than ``mem``. * The powerplants that have been shut down by 2021 are filtered out. -* Updated historical `EIA hydro generation data `_. +* Updated historical `EIA hydro generation data `__. * Network building is made deterministic by supplying a fixed random state to network clustering routines. @@ -1168,24 +1168,24 @@ Synchronisation Release - Ukraine and Moldova (17th March 2022) --------------------------------------------------------------- On March 16, 2022, the transmission networks of Ukraine and Moldova have -successfully been `synchronised with the continental European grid `_. We have taken +successfully been `synchronised with the continental European grid `__. We have taken this as an opportunity to add the power systems of Ukraine and Moldova to PyPSA-Eur. This includes: .. image:: img/synchronisation.png :width: 500 -* the transmission network topology from the `ENTSO-E interactive map `_. +* the transmission network topology from the `ENTSO-E interactive map `__. -* existing power plants (incl. nuclear, coal, gas and hydro) from the `powerplantmatching `_ tool +* existing power plants (incl. nuclear, coal, gas and hydro) from the `powerplantmatching `__ tool -* country-level load time series from ENTSO-E through the `OPSD platform `_, which are then distributed heuristically to substations by GDP and population density. +* country-level load time series from ENTSO-E through the `OPSD platform `__, which are then distributed heuristically to substations by GDP and population density. * wind and solar profiles based on ERA5 and SARAH-2 weather data -* hydro profiles based on historical `EIA generation data `_ +* hydro profiles based on historical `EIA generation data `__ -* a simplified calculation of wind and solar potentials based on the `Copernicus Land Cover dataset `_. +* a simplified calculation of wind and solar potentials based on the `Copernicus Land Cover dataset `__. * electrical characteristics of 750 kV transmission lines @@ -1206,18 +1206,18 @@ PyPSA-Eur 0.4.0 (22th September 2021) * With this release, we change the license from copyleft GPLv3 to the more liberal MIT license with the consent of all contributors - [`#276 `_]. + [`#276 `__]. * Switch to the new major ``atlite`` release v0.2. The version upgrade comes along with significant speed up for the rule ``build_renewable_profiles.py`` (~factor 2). A lot of the code which calculated the land-use availability is now outsourced and does not rely on ``glaes``, ``geokit`` anymore. This facilitates the environment building and version compatibility of ``gdal``, ``libgdal`` with - other packages [`#224 `_]. + other packages [`#224 `__]. * Implemented changes to ``n.snapshot_weightings`` in new PyPSA version v0.18 - (cf. `PyPSA/PyPSA/#227 `_) - [`#259 `_]. + (cf. `PyPSA/PyPSA/#227 `__) + [`#259 `__]. * Add option to pre-aggregate nodes without power injections (positive or negative, i.e. generation or demand) to electrically closest nodes or neighbors @@ -1226,18 +1226,18 @@ PyPSA-Eur 0.4.0 (22th September 2021) * In :mod:`simplify_network`, bus columns with no longer correct entries are removed (symbol, tags, under_construction, substation_lv, substation_off) - [`#219 `_] + [`#219 `__] * Add option to include marginal costs of links representing fuel cells, electrolysis, and battery inverters - [`#232 `_]. + [`#232 `__]. * The rule and script ``build_country_flh`` are removed as they are no longer used or maintained. * The connection cost of generators in :mod:`simplify_network` are now reported in ``resources/connection_costs_s{simpl}.csv`` - [`#261 `_]. + [`#261 `__]. * The tutorial cutout was renamed from ``cutouts/europe-2013-era5.nc`` to ``cutouts/be-03-2013-era5.nc`` to accommodate tutorial and productive @@ -1247,72 +1247,72 @@ PyPSA-Eur 0.4.0 (22th September 2021) potentials was deprecated and now defaults to ``True``. * Update dependencies in ``envs/environment.yaml`` - [`#257 `_] + [`#257 `__] * Continuous integration testing switches to Github Actions from Travis CI - [`#252 `_]. + [`#252 `__]. * Documentation on readthedocs.io is now built with ``pip`` only and no longer - requires ``conda`` [`#267 `_]. + requires ``conda`` [`#267 `__]. -* Use ``Citation.cff`` [`#273 `_]. +* Use ``Citation.cff`` [`#273 `__]. **Bugs and Compatibility** -* Support for PyPSA v0.18 [`#268 `_]. +* Support for PyPSA v0.18 [`#268 `__]. * Minimum Python version set to ``3.8``. -* Removed ``six`` dependency [`#245 `_]. +* Removed ``six`` dependency [`#245 `__]. * Update :mod:`plot_network` and :mod:`make_summary` rules to latest PyPSA - versions [`#270 `_]. + versions [`#270 `__]. * Keep converter links to store components when using the ``ATK`` - wildcard and only remove DC links [`#214 `_]. + wildcard and only remove DC links [`#214 `__]. * Value for ``co2base`` in ``config.yaml`` adjusted to 1.487e9 t CO2-eq (from 3.1e9 t CO2-eq). The new value represents emissions related to the electricity sector for EU+UK+Balkan. The old value was too high and used when the emissions wildcard in ``{opts}`` was used - [`#233 `_]. + [`#233 `__]. * Add escape in :mod:`base_network` if all TYNDP links are already contained in the network - [`#246 `_]. + [`#246 `__]. * In :mod:`solve_operations_network` the optimised capacities are now fixed for all extendable links, not only HVDC links - [`#244 `_]. + [`#244 `__]. * The ``focus_weights`` are now also considered when pre-clustering in the :mod:`simplify_network` rule - [`#241 `_]. + [`#241 `__]. * in :mod:`build_renewable_profile` where offshore wind profiles could - no longer be created [`#249 `_]. + no longer be created [`#249 `__]. * Lower expansion limit of extendable carriers is now set to the existing capacity, i.e. ``p_nom_min = p_nom`` (0 before). Simultaneously, the upper limit (``p_nom_max``) is now the maximum of the installed capacity (``p_nom``) and the previous estimate based on land availability (``p_nom_max``) - [`#260 `_]. + [`#260 `__]. * Solving an operations network now includes optimized store capacities as well. Before only lines, links, generators and storage units were considered - [`#269 `_]. + [`#269 `__]. * With ``load_shedding: true`` in the solving options of ``config.yaml`` load shedding generators are only added at the AC buses, excluding buses for H2 - and battery stores [`#269 `_]. + and battery stores [`#269 `__]. * Delete duplicated capital costs at battery discharge link - [`#240 `_]. + [`#240 `__]. * Propagate the solver log file name to the solver. Previously, the PyPSA network solving functions were not told about the solver logfile specified - in the Snakemake file [`#247 `_] + in the Snakemake file [`#247 `__] PyPSA-Eur 0.3.0 (7th December 2020) ----------------------------------- @@ -1325,29 +1325,29 @@ Using the ``{opts}`` wildcard for scenario: For example ``EQ0.5c`` set in the ``{opts}`` wildcard requires each country to produce on average at least 50% of its consumption. Additionally, the option ``ATK`` requires autarky at each node and removes all means of power transmission through lines and links. ``ATKc`` only removes cross-border transfer capacities. - [`#166 `_]. + [`#166 `__]. * Added an option to alter the capital cost (``c``) or installable potentials (``p``) of carriers by a factor via ``carrier+{c,p}factor`` in the ``{opts}`` wildcard. This can be useful for exploring uncertain cost parameters. Example: ``solar+c0.5`` reduces the capital cost of solar to 50% of original values - [`#167 `_, `#207 `_]. + [`#167 `__, `#207 `__]. * Added an option to the ``{opts}`` wildcard that applies a time series segmentation algorithm based on renewables, hydro inflow and load time series to produce a given total number of adjacent snapshots of varying lengths. This feature is an alternative to downsampling the temporal resolution by simply averaging and - uses the `tsam `_ package - [`#186 `_]. + uses the `tsam `__ package + [`#186 `__]. More OPSD integration: -* Add renewable power plants from `OPSD `_ to the network for specified technologies. +* Add renewable power plants from `OPSD `__ to the network for specified technologies. This will overwrite the capacities calculated from the heuristic approach in :func:`estimate_renewable_capacities()` - [`#212 `_]. + [`#212 `__]. -* Electricity consumption data is now retrieved directly from the `OPSD website `_ using the rule :mod:`build_electricity_demand`. +* Electricity consumption data is now retrieved directly from the `OPSD website `__ using the rule :mod:`build_electricity_demand`. The user can decide whether to take the ENTSO-E power statistics data (default) or the ENTSO-E transparency data - [`#211 `_]. + [`#211 `__]. Other: @@ -1355,93 +1355,93 @@ Other: Then, the rule looks for custom busmaps at ``data/custom_busmap_elec_s{simpl}_{clusters}.csv``, which should have the same format as ``resources/busmap_elec_s{simpl}_{clusters}.csv``. i.e. the index should contain the buses of ``networks/elec_s{simpl}.nc`` - [`#193 `_]. + [`#193 `__]. * Line and link capacities can be capped in the ``config.yaml`` at ``lines: s_nom_max:`` and ``links: p_nom_max``: - [`#166 `_]. + [`#166 `__]. * Added Google Cloud Platform tutorial (for Windows users) - [`#177 `_]. + [`#177 `__]. **Changes** * Don't remove capital costs from lines and links, when imposing a line volume limit (``lv``) or a line cost limit (``lc``). Previously, these were removed to move the expansion in direction of the limit - [`#183 `_]. + [`#183 `__]. * The mappings for clustered lines and buses produced by the :mod:`simplify_network` and :mod:`cluster_network` rules changed from Hierarchical Data Format (``.h5``) to Comma-Separated Values format (``.csv``) for ease of use. - [`#198 `_] + [`#198 `__] * The N-1 security margin for transmission lines is now fixed to a provided value in ``config.yaml``, removing an undocumented linear interpolation between 0.5 and 0.7 in the range between 37 and 200 nodes. - [`#199 `_]. + [`#199 `__]. * Modelling hydrogen and battery storage with Store and Link components is now the default, rather than using StorageUnit components with fixed power-to-energy ratio - [`#205 `_]. + [`#205 `__]. * Use ``mamba`` (https://github.com/mamba-org/mamba) for faster Travis CI builds - [`#196 `_]. + [`#196 `__]. * Multiple smaller changes: Removed unused ``{network}`` wildcard, moved environment files to dedicated ``envs`` folder, removed sector-coupling components from configuration files, updated documentation colors, minor refactoring and code cleaning - [`#190 `_]. + [`#190 `__]. **Bugs and Compatibility** * Add compatibility for pyomo 5.7.0 in :mod:`cluster_network` and :mod:`simplify_network` - [`#172 `_]. + [`#172 `__]. * Fixed a bug for storage units such that individual store and dispatch efficiencies are correctly taken account of rather than only their round-trip efficiencies. In the cost database (``data/costs.csv``) the efficiency of battery inverters should be stated as per discharge/charge rather than per roundtrip - [`#202 `_]. + [`#202 `__]. * Corrected exogenous emission price setting (in ``config: cost: emission price:``), which now correctly accounts for the efficiency and effective emission of the generators - [`#171 `_]. + [`#171 `__]. * Corrected HVDC link connections (a) between Norway and Denmark and (b) mainland Italy, Corsica (FR) and Sardinia (IT) as well as for East-Western and Anglo-Scottish interconnectors - [`#181 `_, `#206 `_]. + [`#181 `__, `#206 `__]. * Fix bug of clustering ``offwind-{ac,dc}`` generators in the option of high-resolution generators for renewables. Now, there are more sites for ``offwind-{ac,dc}`` available than network nodes. Before, they were clustered to the resolution of the network (``elec_s1024_37m.nc``: 37 network nodes, 1024 generators) - [`#191 `_]. + [`#191 `__]. * Raise a warning if ``tech_colors`` in the config are not defined for all carriers - [`#178 `_]. + [`#178 `__]. PyPSA-Eur 0.2.0 (8th June 2020) ------------------------------- -* The optimization is now performed using the ``pyomo=False`` setting in the :func:`pypsa.lopf.network_lopf`. This speeds up the solving process significantly and consumes much less memory. The inclusion of additional constraints were adjusted to the new implementation. They are all passed to the :func:`network_lopf` function via the ``extra_functionality`` argument. The rule ``trace_solve_network`` was integrated into the rule :mod:`solve_network` and can be activated via configuration with ``solving: options: track_iterations: true``. The charging and discharging capacities of batteries modelled as store-link combination are now coupled [`#116 `_]. +* The optimization is now performed using the ``pyomo=False`` setting in the :func:`pypsa.lopf.network_lopf`. This speeds up the solving process significantly and consumes much less memory. The inclusion of additional constraints were adjusted to the new implementation. They are all passed to the :func:`network_lopf` function via the ``extra_functionality`` argument. The rule ``trace_solve_network`` was integrated into the rule :mod:`solve_network` and can be activated via configuration with ``solving: options: track_iterations: true``. The charging and discharging capacities of batteries modelled as store-link combination are now coupled [`#116 `__]. -* An updated extract of the `ENTSO-E Transmission System Map `_ (including Malta) was added to the repository using the `GridKit `_ tool. This tool has been updated to retrieve up-to-date map extracts using a single `script `_. The update extract features 5322 buses, 6574 lines, 46 links. [`#118 `_]. +* An updated extract of the `ENTSO-E Transmission System Map `__ (including Malta) was added to the repository using the `GridKit `__ tool. This tool has been updated to retrieve up-to-date map extracts using a single `script `__. The update extract features 5322 buses, 6574 lines, 46 links. [`#118 `__]. -* Added `FSFE REUSE `_ compliant license information. Documentation now licensed under CC-BY-4.0 [`#160 `_]. +* Added `FSFE REUSE `__ compliant license information. Documentation now licensed under CC-BY-4.0 [`#160 `__]. -* Added a 30 minute `video introduction `_ and a 20 minute `video tutorial `_ +* Added a 30 minute `video introduction `__ and a 20 minute `video tutorial `__ * Networks now store a color and a nicely formatted name for each carrier, accessible via ``n.carrier['color']`` and ``n.carrier['nice_name'] ``(networks after ``elec.nc``). * Added an option to skip iterative solving usually performed to update the line impedances of expanded lines at ``solving: options: skip_iterations:``. -* ``snakemake`` rules for retrieving cutouts and the natura raster can now be disabled independently from their respective rules to build them; via ``config.*yaml`` [`#136 `_]. +* ``snakemake`` rules for retrieving cutouts and the natura raster can now be disabled independently from their respective rules to build them; via ``config.*yaml`` [`#136 `__]. -* Removed the ``id`` column for custom power plants in ``data/custom_powerplants.csv`` to avoid custom power plants with conflicting ids getting attached to the wrong bus [`#131 `_]. +* Removed the ``id`` column for custom power plants in ``data/custom_powerplants.csv`` to avoid custom power plants with conflicting ids getting attached to the wrong bus [`#131 `__]. -* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all available weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed [`#150 `_]. +* Add option ``renewables: {carrier}: keep_all_available_areas:`` to use all available weather cells for renewable profile and potential generation. The default ignores weather cells where only less than 1 MW can be installed [`#150 `__]. -* Added a function ``_helpers.load_network()`` which loads a network with overridden components specified in ``snakemake.config['override_components']`` [`#128 `_]. +* Added a function ``_helpers.load_network()`` which loads a network with overridden components specified in ``snakemake.config['override_components']`` [`#128 `__]. -* Bugfix in :mod:`base_network` which now finds all closest links, not only the first entry [`#143 `_]. +* Bugfix in :mod:`base_network` which now finds all closest links, not only the first entry [`#143 `__]. -* Bugfix in :mod:`cluster_network` which now skips recalculation of link parameters if there are no links [`#149 `_]. +* Bugfix in :mod:`cluster_network` which now skips recalculation of link parameters if there are no links [`#149 `__]. -* Added information on pull requests to contribution guidelines [`#151 `_]. +* Added information on pull requests to contribution guidelines [`#151 `__]. * Improved documentation on open-source solver setup and added usage warnings. @@ -1452,31 +1452,31 @@ PyPSA-Eur 0.1.0 (9th January 2020) This is the first release of PyPSA-Eur, a model of the European power system at the transmission network level. Recent changes include: -* Documentation on installation, workflows and configuration settings is now available online at `pypsa-eur.readthedocs.io `_ [`#65 `_]. +* Documentation on installation, workflows and configuration settings is now available online at `pypsa-eur.readthedocs.io `__ [`#65 `__]. -* The ``conda`` environment files were updated and extended [`#81 `_]. +* The ``conda`` environment files were updated and extended [`#81 `__]. -* The power plant database was updated with extensive filtering options via ``pandas.query`` functionality [`#84 `_ and `#94 `_]. +* The power plant database was updated with extensive filtering options via ``pandas.query`` functionality [`#84 `__ and `#94 `__]. -* Continuous integration testing with `Travis CI `_ is now included for Linux, Mac and Windows [`#82 `_]. +* Continuous integration testing with `Travis CI `__ is now included for Linux, Mac and Windows [`#82 `__]. -* Data dependencies were moved to `zenodo `_ and are now versioned [`#60 `_]. +* Data dependencies were moved to `zenodo `__ and are now versioned [`#60 `__]. -* Data dependencies are now retrieved directly from within the snakemake workflow [`#86 `_]. +* Data dependencies are now retrieved directly from within the snakemake workflow [`#86 `__]. -* Emission prices can be added to marginal costs of generators through the keywords ``Ep`` in the ``{opts}`` wildcard [`#100 `_]. +* Emission prices can be added to marginal costs of generators through the keywords ``Ep`` in the ``{opts}`` wildcard [`#100 `__]. -* An option is introduced to add extendable nuclear power plants to the network [`#98 `_]. +* An option is introduced to add extendable nuclear power plants to the network [`#98 `__]. -* Focus weights can now be specified for particular countries for the network clustering, which allows to set a proportion of the total number of clusters for particular countries [`#87 `_]. +* Focus weights can now be specified for particular countries for the network clustering, which allows to set a proportion of the total number of clusters for particular countries [`#87 `__]. -* A new rule :mod:`add_extra_components` allows to add additional components to the network only after clustering. It is thereby possible to model storage units (e.g. battery and hydrogen) in more detail via a combination of ``Store``, ``Link`` and ``Bus`` elements [`#97 `_]. +* A new rule :mod:`add_extra_components` allows to add additional components to the network only after clustering. It is thereby possible to model storage units (e.g. battery and hydrogen) in more detail via a combination of ``Store``, ``Link`` and ``Bus`` elements [`#97 `__]. -* Hydrogen pipelines (including cost assumptions) can now be added alongside clustered network connections in the rule :mod:`add_extra_components` . Set ``electricity: extendable_carriers: Link: [H2 pipeline]`` and ensure hydrogen storage is modelled as a ``Store``. This is a first simplified stage [`#108 `_]. +* Hydrogen pipelines (including cost assumptions) can now be added alongside clustered network connections in the rule :mod:`add_extra_components` . Set ``electricity: extendable_carriers: Link: [H2 pipeline]`` and ensure hydrogen storage is modelled as a ``Store``. This is a first simplified stage [`#108 `__]. -* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` [`#102 `_]. +* Logfiles for all rules of the ``snakemake`` workflow are now written in the folder ``log/`` [`#102 `__]. -* The new function ``_helpers.mock_snakemake`` creates a ``snakemake`` object which mimics the actual ``snakemake`` object produced by workflow by parsing the ``Snakefile`` and setting all paths for inputs, outputs, and logs. This allows running all scripts within a (I)python terminal (or just by calling ``python ``) and thereby facilitates developing and debugging scripts significantly [`#107 `_]. +* The new function ``_helpers.mock_snakemake`` creates a ``snakemake`` object which mimics the actual ``snakemake`` object produced by workflow by parsing the ``Snakefile`` and setting all paths for inputs, outputs, and logs. This allows running all scripts within a (I)python terminal (or just by calling ``python ``) and thereby facilitates developing and debugging scripts significantly [`#107 `__]. PyPSA-Eur-Sec Releases (pre-merge) @@ -1493,13 +1493,13 @@ biomass, and explicit modelling of methanol and ammonia as separate energy carriers. This release is known to work with `PyPSA-Eur -`_ Version 0.7.0 and `Technology Data -`_ Version 0.5.0. +`__ Version 0.7.0 and `Technology Data +`__ Version 0.5.0. **Gas Transmission Network** * New rule ``retrieve_gas_infrastructure_data`` that downloads and extracts the - SciGRID_gas `IGGIELGN `_ dataset from + SciGRID_gas `IGGIELGN `__ dataset from zenodo. It includes data on the transmission routes, pipe diameters, capacities, pressure, and whether the pipeline is bidirectional and carries H-Gas or L-Gas. @@ -1510,7 +1510,7 @@ This release is known to work with `PyPSA-Eur * New rule ``build_gas_input_locations`` compiles the LNG import capacities (from the Global Energy Monitor's `Europe Gas Tracker - `_, pipeline + `__, pipeline entry capacities and local production capacities for each region of the model. These are the regions where fossil gas can eventually enter the model. @@ -1523,7 +1523,7 @@ This release is known to work with `PyPSA-Eur * With the option ``sector: gas_network:``, the existing gas network is added with a lossless transport model. A length-weighted `k-edge augmentation algorithm - `_ + `__ can be run to add new candidate gas pipelines such that all regions of the model can be connected to the gas network. The number of candidates can be controlled via the setting ``sector: gas_network_connectivity_upgrade:``. When @@ -1554,7 +1554,7 @@ This release is known to work with `PyPSA-Eur * Add option for regionally-resolved geological carbon dioxide sequestration potentials through new rule ``build_sequestration_potentials`` based on - `CO2StoP `_. This + `CO2StoP `__. This can be controlled in the section ``regional_co2_sequestration_potential`` of the ``config.yaml``. It includes options to select the level of conservatism, whether onshore potentials should be included, the respective upper and lower @@ -1566,7 +1566,7 @@ This release is known to work with `PyPSA-Eur ``seq200`` in the ``{sector_opts}`` wildcard (for limit of 200 Mt CO2). * Add option to include `Allam cycle gas power plants - `_ (``allam_cycle``). + `__ (``allam_cycle``). * Add option for planning a new carbon dioxide network (``co2network``). @@ -1585,7 +1585,7 @@ This release is known to work with `PyPSA-Eur * Add regionalised hydrogen salt cavern storage potentials from `Technical Potential of Salt Caverns for Hydrogen Storage in Europe - `_. This data is compiled in + `__. This data is compiled in a new rule ``build_salt_cavern_potentials``. * Add option to resolve ammonia as separate energy carrier with Haber-Bosch @@ -1602,11 +1602,11 @@ This release is known to work with `PyPSA-Eur * Demand for liquid hydrogen in international shipping is now geographically distributed by port trade volumes in a new rule ``build_shipping_demand`` using data from the `World Bank Data Catalogue - `_. + `__. Domestic shipping remains distributed by population. * Add option to aggregate network temporally using representative snapshots or - segments (with `tsam `_). + segments (with `tsam `__). * Add option for minimum part load for Fischer-Tropsch plants (default: 90%) and methanolisation plants (default: 50%). @@ -1659,7 +1659,7 @@ This release is known to work with `PyPSA-Eur PyPSA network. * Updated `data bundle - `_ + `__ that includes the hydrogan salt cavern storage potentials. * Updated and extended documentation in @@ -1687,7 +1687,7 @@ This release is known to work with `PyPSA-Eur expansion of previous iteration as minimum capacity for next iteration. * Further rather minor bugfixes for myopic optimisation code (see `#256 - `_). + `__). Many thanks to all who contributed to this release! @@ -1706,9 +1706,9 @@ more options in setting exogenous transition paths, besides many performance improvements. This release is known to work with `PyPSA-Eur -`_ Version 0.4.0, `Technology Data -`_ Version 0.3.0 and -`PyPSA `_ Version 0.18.0. +`__ Version 0.4.0, `Technology Data +`__ Version 0.3.0 and +`PyPSA `__ Version 0.18.0. Please note that the data bundle has also been updated. @@ -1725,11 +1725,11 @@ Please note that the data bundle has also been updated. battery costs. * Separate basic chemicals into HVC (high-value chemicals), chlorine, methanol and ammonia - [`#166 `_]. + [`#166 `__]. * Add option to specify reuse, primary production, and mechanical and chemical recycling fraction of platics - [`#166 `_]. + [`#166 `__]. * Include energy demands and CO2 emissions for the agriculture, forestry and fishing sector. It is included by default through the option ``A`` in the ``sector_opts`` wildcard. @@ -1743,11 +1743,11 @@ Please note that the data bundle has also been updated. Heat demand is assigned at "services rural heat" buses. Electricity demands are added to low-voltage buses. Time series for demands are constant and distributed inside countries by population - [`#147 `_]. + [`#147 `__]. * Include today's district heating shares in myopic optimisation and add option to specify exogenous path for district heating share increase under ``sector: - district_heating:`` [`#149 `_]. + district_heating:`` [`#149 `__]. * Added option for hydrogen liquefaction costs for hydrogen demand in shipping. This introduces a new ``H2 liquid`` bus at each location. It is activated via @@ -1767,16 +1767,16 @@ Please note that the data bundle has also been updated. factor 2. In this example, ``e_nom_max`` represents the CO2 sequestration potential in Europe. -* Use `JRC ENSPRESO database `_ to +* Use `JRC ENSPRESO database `__ to spatially disaggregate biomass potentials to PyPSA-Eur regions based on overlaps with NUTS2 regions from ENSPRESO (proportional to area) (`#151 - `_). + `__). * Add option to regionally disaggregate biomass potential to individual nodes (previously given per country, then distributed by population density within) and allow the transport of solid biomass. The transport costs are determined based on the `JRC-EU-Times Bioenergy report - `_ in the new optional rule + `__ in the new optional rule ``build_biomass_transport_costs``. Biomass transport can be activated with the setting ``sector: biomass_transport: true``. @@ -1794,7 +1794,7 @@ Please note that the data bundle has also been updated. * The myopic option can now be used together with different clustering for the generators and the network. The existing renewable capacities are split evenly - among the regions in every country [`#144 `_]. + among the regions in every country [`#144 `__]. * Add optional function to use ``geopy`` to locate entries of the Hotmaps database of industrial sites with missing location based on city and country, @@ -1860,7 +1860,7 @@ Please note that the data bundle has also been updated. * Consistent use of ``__main__`` block and further unspecific code cleaning. -* Updated data bundle and moved data bundle to zenodo.org (`10.5281/zenodo.5546517 `_). +* Updated data bundle and moved data bundle to zenodo.org (`10.5281/zenodo.5546517 `__). **Bugfixes and Compatibility** @@ -1884,7 +1884,7 @@ PyPSA-Eur-Sec 0.5.0 (21st May 2021) This release includes improvements to the cost database for building retrofits, carbon budget management and wildcard settings, as well as an important bugfix for the emissions from land transport. -This release is known to work with `PyPSA-Eur `_ Version 0.3.0 and `Technology Data `_ Version 0.2.0. +This release is known to work with `PyPSA-Eur `__ Version 0.3.0 and `Technology Data `__ Version 0.2.0. Please note that the data bundle has also been updated. @@ -1905,15 +1905,15 @@ PyPSA-Eur-Sec 0.4.0 (11th December 2020) This release includes a more accurate nodal disaggregation of industry demand within each country, fixes to CHP and CCS representations, as well as changes to some configuration settings. -It has been released to coincide with `PyPSA-Eur `_ Version 0.3.0 and `Technology Data `_ Version 0.2.0, and is known to work with these releases. +It has been released to coincide with `PyPSA-Eur `__ Version 0.3.0 and `Technology Data `__ Version 0.2.0, and is known to work with these releases. New features: -* The `Hotmaps Industrial Database `_ is used to disaggregate the industrial demand spatially to the nodes inside each country (previously it was distributed by population density). +* The `Hotmaps Industrial Database `__ is used to disaggregate the industrial demand spatially to the nodes inside each country (previously it was distributed by population density). * Electricity demand from industry is now separated from the regular electricity demand and distributed according to the industry demand. Only the remaining regular electricity demand for households and services is distributed according to GDP and population. -* A cost database for the retrofitting of the thermal envelope of residential and services buildings has been integrated, as well as endogenous optimisation of the level of retrofitting. This is described in the paper `Mitigating heat demand peaks in buildings in a highly renewable European energy system `_. Retrofitting can be activated both exogenously and endogenously from the ``config.yaml``. -* The biomass and gas combined heat and power (CHP) parameters ``c_v`` and ``c_b`` were read in assuming they were extraction plants rather than back pressure plants. The data is now corrected in `Technology Data `_ Version 0.2.0 to the correct DEA back pressure assumptions and they are now implemented as single links with a fixed ratio of electricity to heat output (even as extraction plants, they were always sitting on the backpressure line in simulations, so there was no point in modelling the full heat-electricity feasibility polygon). The old assumptions underestimated the heat output. -* The Danish Energy Agency released `new assumptions for carbon capture `_ in October 2020, which have now been incorporated in PyPSA-Eur-Sec, including direct air capture (DAC) and post-combustion capture on CHPs, cement kilns and other industrial facilities. The electricity and heat demand for DAC is modelled for each node (with heat coming from district heating), but currently the electricity and heat demand for industrial capture is not modelled very cleanly (for process heat, 10% of the energy is assumed to go to carbon capture) - a new issue will be opened on this. +* A cost database for the retrofitting of the thermal envelope of residential and services buildings has been integrated, as well as endogenous optimisation of the level of retrofitting. This is described in the paper `Mitigating heat demand peaks in buildings in a highly renewable European energy system `__. Retrofitting can be activated both exogenously and endogenously from the ``config.yaml``. +* The biomass and gas combined heat and power (CHP) parameters ``c_v`` and ``c_b`` were read in assuming they were extraction plants rather than back pressure plants. The data is now corrected in `Technology Data `__ Version 0.2.0 to the correct DEA back pressure assumptions and they are now implemented as single links with a fixed ratio of electricity to heat output (even as extraction plants, they were always sitting on the backpressure line in simulations, so there was no point in modelling the full heat-electricity feasibility polygon). The old assumptions underestimated the heat output. +* The Danish Energy Agency released `new assumptions for carbon capture `__ in October 2020, which have now been incorporated in PyPSA-Eur-Sec, including direct air capture (DAC) and post-combustion capture on CHPs, cement kilns and other industrial facilities. The electricity and heat demand for DAC is modelled for each node (with heat coming from district heating), but currently the electricity and heat demand for industrial capture is not modelled very cleanly (for process heat, 10% of the energy is assumed to go to carbon capture) - a new issue will be opened on this. * Land transport is separated by energy carrier (fossil, hydrogen fuel cell electric vehicle, and electric vehicle), but still needs to be separated into heavy and light vehicles (the data is there, just not the code yet). * For assumptions that change with the investment year, there is a new time-dependent format in the ``config.yaml`` using a dictionary with keys for each year. Implemented examples include the CO2 budget, exogenous retrofitting share and land transport energy carrier; more parameters will be dynamised like this in future. * Some assumptions have been moved out of the code and into the ``config.yaml``, including the carbon sequestration potential and cost, the heat pump sink temperature, reductions in demand for high value chemicals, and some BEV DSM parameters and transport efficiencies. @@ -1936,7 +1936,7 @@ New features: * The script ``build_industrial_production_per_country_tomorrow.py`` determines the future industrial production of materials based on today's levels as well as assumed recycling and demand change measures. * The energy demand for each industry sector and each location in 2015 is also calculated, so that it can be later incorporated in the pathway optimization. * Ammonia production data is taken from the USGS and deducted from JRC-IDEES's "basic chemicals" so that it ammonia can be handled separately from the others (olefins, aromatics and chlorine). -* Solid biomass is no longer allowed to be used for process heat in cement and basic chemicals, since the wastes and residues cannot be guaranteed to reach the high temperatures required. Instead, solid biomass is used in the paper and pulp as well as food, beverages and tobacco industries, where required temperatures are lower (see `DOI:10.1002/er.3436 `_ and `DOI:10.1007/s12053-017-9571-y `_). +* Solid biomass is no longer allowed to be used for process heat in cement and basic chemicals, since the wastes and residues cannot be guaranteed to reach the high temperatures required. Instead, solid biomass is used in the paper and pulp as well as food, beverages and tobacco industries, where required temperatures are lower (see `DOI:10.1002/er.3436 `__ and `DOI:10.1007/s12053-017-9571-y `__). * National installable potentials for salt caverns are now applied. * When electricity distribution grids are activated, new industry electricity demand, resistive heaters and micro-CHPs are now connected to the lower voltage levels. * Gas distribution grid costs are included for gas boilers and micro-CHPs. @@ -1948,15 +1948,15 @@ New features: PyPSA-Eur-Sec 0.2.0 (21st August 2020) -------------------------------------- -This release introduces pathway optimization over many years (e.g. 2020, 2030, 2040, 2050) with myopic foresight, as well as outsourcing the technology assumptions to the `technology-data `_ repository. +This release introduces pathway optimization over many years (e.g. 2020, 2030, 2040, 2050) with myopic foresight, as well as outsourcing the technology assumptions to the `technology-data `__ repository. It is known to work with PyPSA-Eur v0.1.0 (commit bb3477cd69), PyPSA v0.17.1 and technology-data v0.1.0. New features: -* Option for pathway optimization with myopic foresight, based on the paper `Early decarbonisation of the European Energy system pays off (2020) `_. Investments are optimized sequentially for multiple years (e.g. 2020, 2030, 2040, 2050) taking account of existing assets built in previous years and their lifetimes. The script uses data on the existing assets for electricity and building heating technologies, but there are no assumptions yet for existing transport and industry (if you include these, the model will greenfield them). There are also some `outstanding issues `_ on e.g. the distribution of existing wind, solar and heating technologies within each country. To use myopic foresight, set ``foresight : 'myopic'`` in the ``config.yaml`` instead of the default ``foresight : 'overnight'``. An example configuration can be found in ``config.myopic.yaml``. More details on the implementation can be found in :doc:`myopic`. +* Option for pathway optimization with myopic foresight, based on the paper `Early decarbonisation of the European Energy system pays off (2020) `__. Investments are optimized sequentially for multiple years (e.g. 2020, 2030, 2040, 2050) taking account of existing assets built in previous years and their lifetimes. The script uses data on the existing assets for electricity and building heating technologies, but there are no assumptions yet for existing transport and industry (if you include these, the model will greenfield them). There are also some `outstanding issues `__ on e.g. the distribution of existing wind, solar and heating technologies within each country. To use myopic foresight, set ``foresight : 'myopic'`` in the ``config.yaml`` instead of the default ``foresight : 'overnight'``. An example configuration can be found in ``config.myopic.yaml``. More details on the implementation can be found in :doc:`myopic`. -* Technology assumptions (costs, efficiencies, etc.) are no longer stored in the repository. Instead, you have to install the `technology-data `_ database in a parallel directory. These assumptions are largely based on the `Danish Energy Agency Technology Data `_. More details on the installation can be found in :doc:`installation`. +* Technology assumptions (costs, efficiencies, etc.) are no longer stored in the repository. Instead, you have to install the `technology-data `__ database in a parallel directory. These assumptions are largely based on the `Danish Energy Agency Technology Data `__. More details on the installation can be found in :doc:`installation`. * Logs and benchmarks are now stored with the other model outputs in ``results/run-name/``. @@ -1979,7 +1979,7 @@ It is known to work with PyPSA-Eur v0.1.0 (commit bb3477cd69) and PyPSA v0.17.0. We are making this release since in version 0.2.0 we will introduce changes to allow myopic investment planning that will require minor changes for users of the overnight investment planning. PyPSA-Eur-Sec builds on the electricity generation and transmission -model `PyPSA-Eur `_ to add demand +model `PyPSA-Eur `__ to add demand and supply for the following sectors: transport, space and water heating, biomass, industry and industrial feedstocks. This completes the energy system and includes all greenhouse gas emitters except @@ -1988,17 +1988,17 @@ waste management, agriculture, forestry and land use. PyPSA-Eur-Sec was initially based on the model PyPSA-Eur-Sec-30 (Version 0.0.1 below) described in the paper `Synergies of sector coupling and transmission reinforcement in a cost-optimised, highly renewable European energy -system `_ (2018) but it differs by +system `__ (2018) but it differs by being based on the higher resolution electricity transmission model -`PyPSA-Eur `_ rather than a +`PyPSA-Eur `__ rather than a one-node-per-country model, and by including biomass, industry, industrial feedstocks, aviation, shipping, better carbon management, carbon capture and usage/sequestration, and gas networks. PyPSA-Eur-Sec includes PyPSA-Eur as a -`snakemake `_ -`subworkflow `_. PyPSA-Eur-Sec +`snakemake `__ +`subworkflow `__. PyPSA-Eur-Sec uses PyPSA-Eur to build the clustered transmission model along with wind, solar PV and hydroelectricity potentials and time series. Then PyPSA-Eur-Sec adds other conventional generators, storage units and @@ -2013,13 +2013,13 @@ PyPSA-Eur-Sec 0.0.2 (4th September 2020) This version, also called PyPSA-Eur-Sec-30-Path, built on PyPSA-Eur-Sec 0.0.1 (also called PyPSA-Eur-Sec-30) to include myopic pathway optimisation for the paper `Early decarbonisation of the -European energy system pays off `_ +European energy system pays off `__ (2020). The myopic pathway optimisation was then merged into the main PyPSA-Eur-Sec codebase in Version 0.2.0 above. This model has `its own github repository -`_ and is `archived -on Zenodo `_. +`__ and is `archived +on Zenodo `__. @@ -2030,12 +2030,12 @@ This is the first published version of PyPSA-Eur-Sec, also called PyPSA-Eur-Sec-30. It was first used in the research paper `Synergies of sector coupling and transmission reinforcement in a cost-optimised, highly renewable European energy system -`_ (2018). The model covers 30 +`__ (2018). The model covers 30 European countries with one node per country. It includes demand and supply for electricity, space and water heating in buildings, and land transport. -It is `archived on Zenodo `_. +It is `archived on Zenodo `__. Release Process @@ -2058,6 +2058,6 @@ Release Process * Tag a release on Github via ``git tag v0.x.x``, ``git push``, ``git push --tags``. Include release notes in the tag message. -* Make a `GitHub release `_, which automatically triggers archiving to the `zenodo code repository `_ with `MIT license `_. +* Make a `GitHub release `__, which automatically triggers archiving to the `zenodo code repository `__ with `MIT license `__. -* Send announcement on the `PyPSA mailing list `_. +* Send announcement on the `PyPSA mailing list `__. diff --git a/doc/retrieve.rst b/doc/retrieve.rst index f9d6e2a7d..d21a74b48 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -25,12 +25,12 @@ Rule ``retrieve_cutout`` .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.6382570.svg :target: https://doi.org/10.5281/zenodo.6382570 -Cutouts are spatio-temporal subsets of the European weather data from the `ECMWF ERA5 `_ reanalysis dataset and the `CMSAF SARAH-2 `_ solar surface radiation dataset for the year 2013. -They have been prepared by and are for use with the `atlite `_ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo `__ through the rule ``retrieve_cutout``. +Cutouts are spatio-temporal subsets of the European weather data from the `ECMWF ERA5 `__ reanalysis dataset and the `CMSAF SARAH-2 `__ solar surface radiation dataset for the year 2013. +They have been prepared by and are for use with the `atlite `__ tool. You can either generate them yourself using the ``build_cutouts`` rule or retrieve them directly from `zenodo `__ through the rule ``retrieve_cutout``. The :ref:`tutorial` uses a smaller cutout than required for the full model (30 MB), which is also automatically downloaded. .. note:: - To download cutouts yourself from the `ECMWF ERA5 `_ you need to `set up the CDS API `_. + To download cutouts yourself from the `ECMWF ERA5 `__ you need to `set up the CDS API `__. **Relevant Settings** @@ -47,10 +47,10 @@ The :ref:`tutorial` uses a smaller cutout than required for the full model (30 M **Outputs** -- ``cutouts/{cutout}``: weather data from either the `ERA5 `_ reanalysis weather dataset or `SARAH-2 `_ satellite-based historic weather data. +- ``cutouts/{cutout}``: weather data from either the `ERA5 `__ reanalysis weather dataset or `SARAH-2 `__ satellite-based historic weather data. .. seealso:: - For details see :mod:`build_cutout` and read the `atlite documentation `_. + For details see :mod:`build_cutout` and read the `atlite documentation `__. Rule ``retrieve_natura_raster`` @@ -59,7 +59,7 @@ Rule ``retrieve_natura_raster`` .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.4706686.svg :target: https://doi.org/10.5281/zenodo.4706686 -This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff `_) of `Natura 2000 `_ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory. +This rule, as a substitute for :mod:`build_natura_raster`, downloads an already rasterized version (`natura.tiff `__) of `Natura 2000 `__ natural protection areas to reduce computation times. The file is placed into the ``resources`` sub-directory. **Relevant Settings** @@ -74,7 +74,7 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already **Outputs** -- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `_ natural protection areas to reduce computation times. +- ``resources/natura.tiff``: Rasterized version of `Natura 2000 `__ natural protection areas to reduce computation times. .. seealso:: For details see :mod:`build_natura_raster`. @@ -83,7 +83,7 @@ This rule, as a substitute for :mod:`build_natura_raster`, downloads an already Rule ``retrieve_electricity_demand`` ==================================== -This rule downloads hourly electric load data for each country from the `OPSD platform `_. +This rule downloads hourly electric load data for each country from the `OPSD platform `__. **Relevant Settings** @@ -97,7 +97,7 @@ None. Rule ``retrieve_cost_data`` ================================ -This rule downloads techno-economic assumptions from the `technology-data repository `_. +This rule downloads techno-economic assumptions from the `technology-data repository `__. **Relevant Settings** @@ -126,7 +126,7 @@ Rule ``retrieve_irena`` Rule ``retrieve_ship_raster`` ================================ -This rule downloads data on global shipping traffic density from the `World Bank Data Catalogue `_. +This rule downloads data on global shipping traffic density from the `World Bank Data Catalogue `__. **Relevant Settings** diff --git a/doc/sector.rst b/doc/sector.rst index ba3aa3e6e..bdfc5386f 100644 --- a/doc/sector.rst +++ b/doc/sector.rst @@ -7,8 +7,15 @@ Building Sector-Coupled Networks ########################################## -.. warning:: - This part of the documentation is under development. +The preparation process of the sector-coupled version of the PyPSA-Eur energy system model consists of a group of ``snakemake`` rules which are briefly outlined and explained in detail in the sections below. + +Not all data dependencies are shipped with the git repository. +Instead we provide separate data bundles which can be obtained +using the ``retrieve*`` rules (:ref:`data`). +Having downloaded the necessary data, + +- :mod:`add_brownfield` builds and stores the base network with all buses, HVAC lines and HVDC links, while + Rule ``add_brownfield`` ============================================================================== diff --git a/doc/simplification.rst b/doc/simplification.rst index 2272505d6..cb0484617 100644 --- a/doc/simplification.rst +++ b/doc/simplification.rst @@ -12,11 +12,11 @@ Simplifying Electricity Networks The simplification ``snakemake`` rules prepare **approximations** of the full model, for which it is computationally viable to co-optimize generation, storage and transmission capacities. - :mod:`simplify_network` transforms the transmission grid to a 380 kV only equivalent network, while -- :mod:`cluster_network` uses a `k-means `_ based clustering technique to partition the network into a given number of zones and then reduce the network to a representation with one bus per zone. +- :mod:`cluster_network` uses a `k-means `__ based clustering technique to partition the network into a given number of zones and then reduce the network to a representation with one bus per zone. The simplification and clustering steps are described in detail in the paper -- Jonas Hörsch and Tom Brown. `The role of spatial scale in joint optimisations of generation and transmission for European highly renewable scenarios `_), *14th International Conference on the European Energy Market*, 2017. `arXiv:1705.07617 `_, `doi:10.1109/EEM.2017.7982024 `_. +- Jonas Hörsch and Tom Brown. `The role of spatial scale in joint optimisations of generation and transmission for European highly renewable scenarios `__), *14th International Conference on the European Energy Market*, 2017. `arXiv:1705.07617 `__, `doi:10.1109/EEM.2017.7982024 `__. After simplification and clustering of the network, additional components may be appended in the rule :mod:`add_extra_components` and the network is prepared for solving in :mod:`prepare_network`. diff --git a/doc/spatial_resolution.rst b/doc/spatial_resolution.rst index a408f4644..20158ab60 100644 --- a/doc/spatial_resolution.rst +++ b/doc/spatial_resolution.rst @@ -9,7 +9,7 @@ Spatial resolution ########################################## -The default nodal resolution of the model follows the electricity generation and transmission model `PyPSA-Eur `_, which clusters down the electricity transmission substations in each European country based on the k-means algorithm (See `cluster_network `_ for a complete explanation). This gives nodes which correspond to major load and generation centres (typically cities). +The default nodal resolution of the model follows the electricity generation and transmission model `PyPSA-Eur `__, which clusters down the electricity transmission substations in each European country based on the k-means algorithm (See `cluster_network `__ for a complete explanation). This gives nodes which correspond to major load and generation centres (typically cities). The total number of nodes for Europe is set in the ``config/config.yaml`` file under ``clusters``. The number of nodes can vary between 37, the number of independent countries / synchronous areas, and several hundred. With 200-300 nodes the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi. @@ -21,7 +21,7 @@ Exemplary unsolved network clustered to 37 nodes: .. image:: ../graphics/elec_s_37.png -The total number of nodes for Europe is set in the ``config/config.yaml`` file under `clusters `_. The number of nodes can vary between 37, the number of independent countries/synchronous areas, and several hundred. With 200-300 nodes, the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi. +The total number of nodes for Europe is set in the ``config/config.yaml`` file under `clusters `__. The number of nodes can vary between 37, the number of independent countries/synchronous areas, and several hundred. With 200-300 nodes, the model needs 100-150 GB RAM to solve with a commercial solver like Gurobi. Not all of the sectors are at the full nodal resolution, and some demand for some sectors is distributed to nodes using heuristics that need to be corrected. Some networks are copper-plated to reduce computational times. Here are some examples of how spatial resolution is set for different sectors in PyPSA-Eur-Sec: @@ -37,18 +37,18 @@ Here are some examples of how spatial resolution is set for different sectors in • Electricity demand in industry: Modeled as nodal, based on the location of industrial facilities from HotMaps database. • Industry demand (heat, chemicals, etc.) : Modeled as nodal, distributed in each country based on locations of industry from HotMaps database. -• Hydrogen network: Modeled as nodal (if activated in the `config `_ file). +• Hydrogen network: Modeled as nodal (if activated in the `config `__ file). -• Methane network: It can be modeled as a single node for Europe or it can be nodally resolved if activated in the `config `_. One node can be considered reasonable since future demand is expected to be low and no bottlenecks are expected. Also, the nodally resolved methane grid is based on SciGRID_gas data. +• Methane network: It can be modeled as a single node for Europe or it can be nodally resolved if activated in the `config `__. One node can be considered reasonable since future demand is expected to be low and no bottlenecks are expected. Also, the nodally resolved methane grid is based on SciGRID_gas data. -• Solid biomass: It can be modeled as a single node for Europe or it can be nodally resolved if activated in the `config `_. Nodal modeling includes modeling biomass potential per country (given per country, then distributed by population density within) and the transport of solid biomass between countries. +• Solid biomass: It can be modeled as a single node for Europe or it can be nodally resolved if activated in the `config `__. Nodal modeling includes modeling biomass potential per country (given per country, then distributed by population density within) and the transport of solid biomass between countries. -• CO2: It can be modeled as a single node for Europe or it can be nodally resolved with CO2 transport pipelines if activated in the `config `_. It should mentioned that in single node mode a transport and storage cost is added for sequestered CO2, the cost of which can be adjusted in the `config `_. +• CO2: It can be modeled as a single node for Europe or it can be nodally resolved with CO2 transport pipelines if activated in the `config `__. It should mentioned that in single node mode a transport and storage cost is added for sequestered CO2, the cost of which can be adjusted in the `config `__. • Carbonaceous fuels: Modeled as a single node for Europe by default, since transport costs for liquids are low and no bottlenecks are expected. Can be regionally resolved in configuration. **Electricity distribution network** -Contrary to the transmission grid, the grid topology at the distribution level (at and below 110 kV) is not included due to the very high computational burden. However, a link per node can be used (if activated in the `Config `_ file) to represent energy transferred between distribution and transmission levels at every node. In essence, the total energy capacity connecting the transmission grid and the low-voltage level is optimized. The cost assumptions for this link can be adjusted in Config file `options `_ , and is currently assumed to be 500 Eur/kW. +Contrary to the transmission grid, the grid topology at the distribution level (at and below 110 kV) is not included due to the very high computational burden. However, a link per node can be used (if activated in the `Config `__ file) to represent energy transferred between distribution and transmission levels at every node. In essence, the total energy capacity connecting the transmission grid and the low-voltage level is optimized. The cost assumptions for this link can be adjusted in Config file `options `__ , and is currently assumed to be 500 Eur/kW. Rooftop PV, heat pumps, resistive heater, home batteries chargers for passenger EVs, as well as individual heating technologies (heat pumps and resistive heaters) are connected to low-voltage level. All the remaining generation and storage technologies are connected to the transmission grid. In practice, this means that the distribution grid capacity is only extended if it is necessary to balance the mismatch between local generation and demand. diff --git a/doc/supply_demand.rst b/doc/supply_demand.rst index 8f6edcad9..cc598aafd 100644 --- a/doc/supply_demand.rst +++ b/doc/supply_demand.rst @@ -11,7 +11,7 @@ An initial orientation to the supply and demand options in the model PyPSA-Eur-Sec can be found in the description of the model PyPSA-Eur-Sec-30 in the paper `Synergies of sector coupling and transmission reinforcement in a cost-optimised, highly renewable -European energy system `_ (2018). +European energy system `__ (2018). The latest version of PyPSA-Eur-Sec differs by including biomass, industry, industrial feedstocks, aviation, shipping, better carbon management, carbon capture and usage/sequestration, and gas networks. @@ -26,13 +26,13 @@ Electricity supply and demand ============================= Electricity supply and demand follows the electricity generation and -transmission model `PyPSA-Eur `_, +transmission model `PyPSA-Eur `__, except that hydrogen storage is integrated into the hydrogen supply, demand and network, and PyPSA-Eur-Sec includes CHPs. Unlike PyPSA-Eur, PyPSA-Eur-Sec does not distribution electricity demand for industry according to population and GDP, but uses the geographical data from the `Hotmaps Industrial Database -`_. +`__. Also unlike PyPSA-Eur, PyPSA-Eur-Sec subtracts existing electrified heating from the existing electricity demand, so that power-to-heat can be optimised separately. @@ -44,7 +44,7 @@ Heat demand =========== Building heating in residential and services sectors is resolved regionally, both for individual buildings and district heating systems, which include different supply options (see :ref:`heat-supply`.) -Annual heat demands per country are retrieved from `JRC-IDEES `_ and split into space and water heating. For space heating, the annual demands are converted to daily values based on the population-weighted Heating Degree Day (HDD) using the `atlite tool `_, where space heat demand is proportional to the difference between the daily average ambient temperature (read from `ERA5 `_) and a threshold temperature above which space heat demand is zero. A threshold temperature of 15 °C is assumed by default. The daily space heat demand is distributed to the hours of the day following heat demand profiles from `BDEW `_. These differ for weekdays and weekends/holidays and between residential and services demand. +Annual heat demands per country are retrieved from `JRC-IDEES `__ and split into space and water heating. For space heating, the annual demands are converted to daily values based on the population-weighted Heating Degree Day (HDD) using the `atlite tool `__, where space heat demand is proportional to the difference between the daily average ambient temperature (read from `ERA5 `__) and a threshold temperature above which space heat demand is zero. A threshold temperature of 15 °C is assumed by default. The daily space heat demand is distributed to the hours of the day following heat demand profiles from `BDEW `__. These differ for weekdays and weekends/holidays and between residential and services demand. *Space heating* @@ -54,11 +54,11 @@ The space heating demand can be exogenously reduced by retrofitting measures tha :language: yaml :lines: 205 -Co-optimsing of building renovation is also possible, if it is activated in the `config file `_. +Co-optimsing of building renovation is also possible, if it is activated in the `config file `__. Renovation of the thermal envelope reduces the space heating demand and is optimised at each node for every heat bus. Renovation measures through additional insulation material and replacement of energy inefficient windows are considered. -In a first step, costs per energy savings are estimated in `build_retro_cost.py `_. They depend on the insulation condition of the building stock and costs for renovation of the building elements. In a second step, for those cost per energy savings two possible renovation strengths are determined: a moderate renovation with lower costs, a lower maximum possible space heat savings, and an ambitious renovation with associated higher costs and higher efficiency gains. They are added by step-wise linearisation in form of two additional generations in `prepare_sector_network.py `_. +In a first step, costs per energy savings are estimated in `build_retro_cost.py `__. They depend on the insulation condition of the building stock and costs for renovation of the building elements. In a second step, for those cost per energy savings two possible renovation strengths are determined: a moderate renovation with lower costs, a lower maximum possible space heat savings, and an ambitious renovation with associated higher costs and higher efficiency gains. They are added by step-wise linearisation in form of two additional generations in `prepare_sector_network.py `__. Further information are given in the publication : - `Mitigating heat demand peaks in buildings in a highly renewable European energy system, (2021) `_. +`Mitigating heat demand peaks in buildings in a highly renewable European energy system, (2021) `__. *Water heating* @@ -66,7 +66,7 @@ Hot water demand is assumed to be constant throughout the year. *Urban and rural heating* -For every country, heat demand is split between low and high population density areas. These country-level totals are then distributed to each region in proportion to their rural and urban populations respectively. Urban areas with dense heat demand can be supplied with large-scale district heating systems. The percentage of urban heat demand that can be supplied by district heating networks as well as lump-sum losses in district heating systems is exogenously determined in the `config file `_. +For every country, heat demand is split between low and high population density areas. These country-level totals are then distributed to each region in proportion to their rural and urban populations respectively. Urban areas with dense heat demand can be supplied with large-scale district heating systems. The percentage of urban heat demand that can be supplied by district heating networks as well as lump-sum losses in district heating systems is exogenously determined in the `config file `__. *Cooling demand* @@ -96,41 +96,41 @@ Different supply options are available depending on whether demand is met centra **Urban central heat** -For large-scale district heating systems the following options are available: combined heat and power (CHP) plants consuming gas or biomass from waste and residues with and without carbon capture (CC), large-scale air-sourced heat pumps, gas and oil boilers, resistive heaters, and fuel cell CHPs. Additionally, waste heat from the `Fischer-Tropsch `_ and `Sabatier `_ processes for the production of synthetic hydrocarbons can supply district heating systems. For more detailed explanation of these processes, see :ref:`Oil-based products supply` and :ref:`Methane supply`. +For large-scale district heating systems the following options are available: combined heat and power (CHP) plants consuming gas or biomass from waste and residues with and without carbon capture (CC), large-scale air-sourced heat pumps, gas and oil boilers, resistive heaters, and fuel cell CHPs. Additionally, waste heat from the `Fischer-Tropsch `__ and `Sabatier `__ processes for the production of synthetic hydrocarbons can supply district heating systems. For more detailed explanation of these processes, see :ref:`Oil-based products supply` and :ref:`Methane supply`. **Residential and Urban decentral heat** Supply options in individual buildings include gas and oil boilers, air- and ground-sourced heat pumps, resistive heaters, and solar thermal collectors. -Ground-source heat pumps are only allowed in rural areas because of space constraints. Thus, only air- source heat pumps are allowed in urban areas. This is a conservative assumption, since there are many possible sources of low-temperature heat that could be tapped in cities (e.g. waste water, ground water, or natural bodies of water). Costs, lifetimes and efficiencies for these technologies are retrieved from the `technology-data repository `_. +Ground-source heat pumps are only allowed in rural areas because of space constraints. Thus, only air- source heat pumps are allowed in urban areas. This is a conservative assumption, since there are many possible sources of low-temperature heat that could be tapped in cities (e.g. waste water, ground water, or natural bodies of water). Costs, lifetimes and efficiencies for these technologies are retrieved from the `technology-data repository `__. -Below are more detailed explanations for each heating supply component, all of which are modelled as `links `_ in PyPSA-Eur-Sec. +Below are more detailed explanations for each heating supply component, all of which are modelled as `links `__ in PyPSA-Eur-Sec. .. _Large-scale CHP: **Large-scale CHP** -Large Combined Heat and Power plants are included in the model if it is specified in the `config file `_. +Large Combined Heat and Power plants are included in the model if it is specified in the `config file `__. -CHPs are based on back pressure plants operating with a fixed ratio of electricity to heat output. The efficiencies of each are given on the back pressure line, where the back pressure coefficient cb is the electricity output divided by the heat output. (For a more complete explanation of the operation of CHPs refer to the study by Dahl et al. : `Cost sensitivity of optimal sector-coupled district heating production systems `_. +CHPs are based on back pressure plants operating with a fixed ratio of electricity to heat output. The efficiencies of each are given on the back pressure line, where the back pressure coefficient cb is the electricity output divided by the heat output. (For a more complete explanation of the operation of CHPs refer to the study by Dahl et al. : `Cost sensitivity of optimal sector-coupled district heating production systems `__. PyPSA-Eur-Sec includes CHP plants fueled by methane and solid biomass from waste and residues. Hydrogen fuel cells also produce both electricity and heat. -The methane CHP is modeled on the Danish Energy Agency (DEA) “Gas turbine simple cycle (large)” while the solid biomass CHP is based on the DEA’s “09b Wood Pellets Medium”. For biomass CHP, cb = `0.46 `_ , whereas for gas CHP, cb = `1 `_. +The methane CHP is modeled on the Danish Energy Agency (DEA) “Gas turbine simple cycle (large)” while the solid biomass CHP is based on the DEA’s “09b Wood Pellets Medium”. For biomass CHP, cb = `0.46 `__ , whereas for gas CHP, cb = `1 `__. -NB: The old PyPSA-Eur-Sec-30 model assumed an extraction plant (like the DEA coal CHP) for gas which has flexible production of heat and electricity within the feasibility diagram of Figure 4 in the study by `Brown et al. `_ We have switched to the DEA back pressure plants since these are more common for smaller plants for biomass, and because the extraction plants were on the back pressure line for 99.5% of the time anyway. The plants were all changed to back pressure in PyPSA-Eur-Sec v0.4.0. +NB: The old PyPSA-Eur-Sec-30 model assumed an extraction plant (like the DEA coal CHP) for gas which has flexible production of heat and electricity within the feasibility diagram of Figure 4 in the study by `Brown et al. `__ We have switched to the DEA back pressure plants since these are more common for smaller plants for biomass, and because the extraction plants were on the back pressure line for 99.5% of the time anyway. The plants were all changed to back pressure in PyPSA-Eur-Sec v0.4.0. **Micro-CHP** -PyPSA-Eur-Sec allows individual buildings to make use of `micro gas CHPs `_ that are assumed to be installed at the distribution grid level. +PyPSA-Eur-Sec allows individual buildings to make use of `micro gas CHPs `__ that are assumed to be installed at the distribution grid level. **Heat pumps** -The coefficient of performance (COP) of air- and ground-sourced heat pumps depends on the ambient or soil temperature respectively. Hence, the COP is a time-varying parameter (refer to `Config `_ file). Generally, the COP will be lower during winter when temperatures are low. Because the ambient temperature is more volatile than the soil temperature, the COP of ground-sourced heat pumps is less variable. Moreover, the COP depends on the difference between the source and sink temperatures: +The coefficient of performance (COP) of air- and ground-sourced heat pumps depends on the ambient or soil temperature respectively. Hence, the COP is a time-varying parameter (refer to `Config `__ file). Generally, the COP will be lower during winter when temperatures are low. Because the ambient temperature is more volatile than the soil temperature, the COP of ground-sourced heat pumps is less variable. Moreover, the COP depends on the difference between the source and sink temperatures: .. math:: \Delta T = T_{sink} − T_{source} -For the sink water temperature Tsink we assume 55 °C [`Config `_ file]. For the time- and location-dependent source temperatures Tsource, we rely on the `ERA5 `_ reanalysis weather data. The temperature differences are converted into COP time series using results from a regression analysis performed in the study by `Stafell et al. `_. For air-sourced heat pumps (ASHP), we use the function: +For the sink water temperature Tsink we assume 55 °C [`Config `__ file]. For the time- and location-dependent source temperatures Tsource, we rely on the `ERA5 `__ reanalysis weather data. The temperature differences are converted into COP time series using results from a regression analysis performed in the study by `Stafell et al. `__. For air-sourced heat pumps (ASHP), we use the function: .. math:: COP (\Delta T) = 6.81 - 0.121\Delta T + 0.000630\Delta T^2 @@ -142,44 +142,44 @@ for ground-sourced heat pumps (GSHP), we use the function: **Resistive heaters** -Can be activated in Config from the `boilers `_ option. -Resistive heaters produce heat with a fixed conversion efficiency (refer to `Technology-data repository `_ ). +Can be activated in Config from the `boilers `__ option. +Resistive heaters produce heat with a fixed conversion efficiency (refer to `Technology-data repository `__ ). **Gas, oil, and biomass boilers** -Can be activated in Config from the `boilers `_ , `oil boilers `_ , and `biomass boiler `_ option. +Can be activated in Config from the `boilers `__ , `oil boilers `__ , and `biomass boiler `__ option. Similar to resistive heaters, boilers have a fixed efficiency and produce heat using gas, oil or biomass. **Solar thermal collectors** -Can be activated in the config file from the `solar_thermal `_ option. -Solar thermal profiles are built based on weather data and also have the `options `_ for setting the sky model and the orientation of the panel in the config file, which are then used by the atlite tool to calculate the solar resource time series. +Can be activated in the config file from the `solar_thermal `__ option. +Solar thermal profiles are built based on weather data and also have the `options `__ for setting the sky model and the orientation of the panel in the config file, which are then used by the atlite tool to calculate the solar resource time series. **Waste heat from Fuel Cells, Methanation and Fischer-Tropsch plants** -Waste heat from `fuel cells `_ in addition to processes like `Fischer-Tropsch `_, methanation, and Direct Air Capture (DAC) is dumped into district heating networks. +Waste heat from `fuel cells `__ in addition to processes like `Fischer-Tropsch `__, methanation, and Direct Air Capture (DAC) is dumped into district heating networks. **Existing heating capacities and decommissioning** -For the myopic transition paths, capacities already existing for technologies supplying heat are retrieved from `“Mapping and analyses of the current and future (2020 - 2030)” `_ . For the sake of simplicity, coal, oil and gas boiler capacities are assimilated to gas boilers. Besides that, existing capacities for heat resistors, air-sourced and ground-sourced heat pumps are included in the model. For heating capacities, 25% of existing capacities in 2015 are assumed to be decommissioned in every 5-year time step after 2020. +For the myopic transition paths, capacities already existing for technologies supplying heat are retrieved from `“Mapping and analyses of the current and future (2020 - 2030)” `__ . For the sake of simplicity, coal, oil and gas boiler capacities are assimilated to gas boilers. Besides that, existing capacities for heat resistors, air-sourced and ground-sourced heat pumps are included in the model. For heating capacities, 25% of existing capacities in 2015 are assumed to be decommissioned in every 5-year time step after 2020. **Thermal Energy Storage** -Activated in Config from the `tes `_ option. +Activated in Config from the `tes `__ option. -Thermal energy can be stored in large water pits associated with district heating systems and individual thermal energy storage (TES), i.e., small water tanks. Water tanks are modelled as `stores `_. -A thermal energy density of 46.8 kWh :math:`_{th}`/m3 is assumed, corresponding to a temperature difference of 40 K. The decay of thermal energy in the stores: 1- :math:`e^{-1/24τ}` is assumed to have a time constant of  τ=180 days for central TES and  τ=3 days for individual TES, both modifiable through `tes_tau `_ in config file. Charging and discharging efficiencies are 90% due to pipe losses. +Thermal energy can be stored in large water pits associated with district heating systems and individual thermal energy storage (TES), i.e., small water tanks. Water tanks are modelled as `stores `__. +A thermal energy density of 46.8 kWh :math:`_{th}`/m3 is assumed, corresponding to a temperature difference of 40 K. The decay of thermal energy in the stores: 1- :math:`e^{-1/24τ}` is assumed to have a time constant of  τ=180 days for central TES and  τ=3 days for individual TES, both modifiable through `tes_tau `__ in config file. Charging and discharging efficiencies are 90% due to pipe losses. **Retrofitting of the thermal envelope of buildings** -Co-optimising building renovation is only enabled if in the `config `_ file. To reduce the computational burden, +Co-optimising building renovation is only enabled if in the `config `__ file. To reduce the computational burden, default setting is set as false. Renovation of the thermal envelope reduces the space heating demand and is optimised at each node for every heat bus. Renovation measures through additional insulation material and replacement of energy inefficient windows are considered. -In a first step, costs per energy savings are estimated in the `build_retro_cost.py `_ script. +In a first step, costs per energy savings are estimated in the `build_retro_cost.py `__ script. They depend on the insulation condition of the building stock and costs for renovation of the building elements. In a second step, for those cost per energy savings two possible renovation @@ -187,12 +187,12 @@ strengths are determined: a moderate renovation with lower costs and lower maximum possible space heat savings, and an ambitious renovation with associated higher costs and higher efficiency gains. They are added by step-wise linearisation in form of two additional generations in -the `prepare_sector_network.py `_ script. +the `prepare_sector_network.py `__ script. Settings in the ``config/config.yaml`` concerning the endogenously optimisation of building -renovation include `cost factor `_, `interest rate `_, `annualised cost `_, `tax weighting `_, and `construction index `_. +renovation include `cost factor `__, `interest rate `__, `annualised cost `__, `tax weighting `__, and `construction index `__. -Further information are given in the study by Zeyen et al. : `Mitigating heat demand peaks in buildings in a highly renewable European energy system, (2021) `_. +Further information are given in the study by Zeyen et al. : `Mitigating heat demand peaks in buildings in a highly renewable European energy system, (2021) `__. .. _Hydrogen demand: @@ -200,7 +200,7 @@ Hydrogen demand ============================= Hydrogen is consumed in the industry sector (see :ref:`Industry demand`) to produce ammonia (see :ref:`Chemicals Industry`) and direct reduced iron (DRI) (see :ref:`Iron and Steel`). Hydrogen is also consumed to produce synthetic methane (see :ref:`Methane supply`) and liquid hydrocarbons (see :ref:`Oil-based products supply`) which have multiple uses in industry and other sectors. -Hydrogen is also used for transport applications (see :ref:`Transportation`), where it is exogenously fixed. It is used in `heavy-duty land transport `_ and as liquified hydrogen in the shipping sector (see :ref:`Shipping`). Furthermore, stationary fuel cells may re-electrify hydrogen (with waste heat as a byproduct) to balance renewable fluctuations (see :ref:`Electricity supply and demand`). The waste heat from the stationary fuel cells can be used in `district-heating systems `_. +Hydrogen is also used for transport applications (see :ref:`Transportation`), where it is exogenously fixed. It is used in `heavy-duty land transport `__ and as liquified hydrogen in the shipping sector (see :ref:`Shipping`). Furthermore, stationary fuel cells may re-electrify hydrogen (with waste heat as a byproduct) to balance renewable fluctuations (see :ref:`Electricity supply and demand`). The waste heat from the stationary fuel cells can be used in `district-heating systems `__. .. _Hydrogen supply: @@ -220,7 +220,7 @@ combined with a water-gas shift reaction CO + H_2O \xrightarrow{} CO_2 + H_2 -SMR is included `here `_. +SMR is included `here `__. PyPSA-Eur-Sec allows this route of :math:`H_2` production with and without [carbon capture (CC)] (see :ref:`Carbon dioxide capture, usage and sequestration (CCU/S)`). These routes are often referred to as blue and grey hydrogen. Here, methane input can be both of fossil or synthetic origin. Green hydrogen can be produced by electrolysis to split water into hydrogen and oxygen @@ -234,12 +234,12 @@ For the electrolysis, alkaline electrolysers are chosen since they have lower co **Transport** -Hydrogen is transported by pipelines. :math:`H_2` pipelines are endogenously generated, either via a greenfield :math:`H_2` network, or by `retrofitting natural gas pipelines `_). Retrofitting is implemented in such a way that for every unit of decommissioned gas pipeline, a share (60% is used in the study by `Neumann et al. `_) of its nominal capacity (exogenously determined in the `config file `_.) is available for hydrogen transport. When the gas network is not resolved, this input denotes the potential for gas pipelines repurposed into hydrogen pipelines. +Hydrogen is transported by pipelines. :math:`H_2` pipelines are endogenously generated, either via a greenfield :math:`H_2` network, or by `retrofitting natural gas pipelines `__). Retrofitting is implemented in such a way that for every unit of decommissioned gas pipeline, a share (60% is used in the study by `Neumann et al. `__) of its nominal capacity (exogenously determined in the `config file `__.) is available for hydrogen transport. When the gas network is not resolved, this input denotes the potential for gas pipelines repurposed into hydrogen pipelines. New pipelines can be built additionally on all routes where there currently is a gas or electricity network connection. These new pipelines will be built where no sufficient retrofitting options are available. The capacities of new and repurposed pipelines are a result of the optimisation. **Storage** -Hydrogen can be stored in overground steel tanks or `underground salt caverns `_. For the latter, energy storage capacities in every country are limited to the potential estimation for onshore salt caverns within `50 km `_ of shore to avoid environmental issues associated with brine solution disposal. Underground storage potentials for hydrogen in European salt caverns is acquired from `Caglayan et al. `_ +Hydrogen can be stored in overground steel tanks or `underground salt caverns `__. For the latter, energy storage capacities in every country are limited to the potential estimation for onshore salt caverns within `50 km `__ of shore to avoid environmental issues associated with brine solution disposal. Underground storage potentials for hydrogen in European salt caverns is acquired from `Caglayan et al. `__ .. _Methane demand: @@ -253,7 +253,7 @@ Methane is used in individual and large-scale gas boilers, in CHP plants with an Methane supply =================================== -In addition to methane from fossil origins, the model also considers biogenic and synthetic sources. `The gas network can either be modelled, or it can be assumed that gas transport is not limited `_. If gas infrastructure is regionally resolved, fossil gas can enter the system only at existing and planned LNG terminals, pipeline entry-points, and intra- European gas extraction sites, which are retrieved from the SciGRID Gas IGGIELGN dataset and the GEM Wiki. +In addition to methane from fossil origins, the model also considers biogenic and synthetic sources. `The gas network can either be modelled, or it can be assumed that gas transport is not limited `__. If gas infrastructure is regionally resolved, fossil gas can enter the system only at existing and planned LNG terminals, pipeline entry-points, and intra- European gas extraction sites, which are retrieved from the SciGRID Gas IGGIELGN dataset and the GEM Wiki. Biogas can be upgraded to methane. Synthetic methane can be produced by processing hydrogen and captures :math:`CO_2` in the Sabatier reaction @@ -275,7 +275,7 @@ The following figure shows the unclustered European gas transmission network bas Biomass Supply ===================== -Biomass supply potentials for each European country are taken from the `JRC ENSPRESO database `_ where data is available for various years (2010, 2020, 2030, 2040 and 2050) and scenarios (low, medium, high). No biomass import from outside Europe is assumed. More information on the data set can be found `here `_. +Biomass supply potentials for each European country are taken from the `JRC ENSPRESO database `__ where data is available for various years (2010, 2020, 2030, 2040 and 2050) and scenarios (low, medium, high). No biomass import from outside Europe is assumed. More information on the data set can be found `here `__. .. _Biomass demand: @@ -283,19 +283,19 @@ Biomass demand ===================== -Biomass supply potentials for every NUTS2 region are taken from the `JRC ENSPRESO database `_ where data is available for various years (2010, 2020, 2030, 2040 and 2050) and different availability scenarios (low, medium, high). No biomass import from outside Europe is assumed. More information on the data set can be found `here `_. The data for NUTS2 regions is mapped to PyPSA-Eur-Sec model regions in proportion to the area overlap. +Biomass supply potentials for every NUTS2 region are taken from the `JRC ENSPRESO database `__ where data is available for various years (2010, 2020, 2030, 2040 and 2050) and different availability scenarios (low, medium, high). No biomass import from outside Europe is assumed. More information on the data set can be found `here `__. The data for NUTS2 regions is mapped to PyPSA-Eur-Sec model regions in proportion to the area overlap. -The desired scenario can be selected in the PyPSA-Eur-Sec `configuration `_. The script for building the biomass potentials from the JRC ENSPRESO data base is located `here `_. Consult the script to see the keywords that specify the scenario options. +The desired scenario can be selected in the PyPSA-Eur-Sec `configuration `__. The script for building the biomass potentials from the JRC ENSPRESO data base is located `here `__. Consult the script to see the keywords that specify the scenario options. -The `configuration `_ also allows the user to define how the various types of biomass are used in the model by using the following categories: biogas, solid biomass, and not included. Feedstocks categorized as biogas, typically manure and sludge waste, are available to the model as biogas, which can be upgraded to biomethane. Feedstocks categorized as solid biomass, e.g. secondary forest residues or municipal waste, are available for combustion in combined-heat-and power (CHP) plants and for medium temperature heat (below 500 °C) applications in industry. It can also converted to gas or liquid fuels. +The `configuration `__ also allows the user to define how the various types of biomass are used in the model by using the following categories: biogas, solid biomass, and not included. Feedstocks categorized as biogas, typically manure and sludge waste, are available to the model as biogas, which can be upgraded to biomethane. Feedstocks categorized as solid biomass, e.g. secondary forest residues or municipal waste, are available for combustion in combined-heat-and power (CHP) plants and for medium temperature heat (below 500 °C) applications in industry. It can also converted to gas or liquid fuels. Feedstocks labeled as not included are ignored by the model. -A `typical use case for biomass `_ would be the medium availability scenario for 2030 where only residues from agriculture and forestry as well as biodegradable municipal waste are considered as energy feedstocks. Fuel crops are avoided because they compete with scarce land for food production, while primary wood, as well as wood chips and pellets, are avoided because of concerns about sustainability. See the supporting materials of the `paper `_ for more details. +A `typical use case for biomass `__ would be the medium availability scenario for 2030 where only residues from agriculture and forestry as well as biodegradable municipal waste are considered as energy feedstocks. Fuel crops are avoided because they compete with scarce land for food production, while primary wood, as well as wood chips and pellets, are avoided because of concerns about sustainability. See the supporting materials of the `paper `__ for more details. *Solid biomass conversion and use* @@ -303,19 +303,19 @@ A `typical use case for biomass `_ would be th Solid biomass can be used directly to provide process heat up to 500˚C in the industry. It can also be burned in CHP plants and boilers associated with heating systems. These technologies are described elsewhere (see :ref:`Large-scale CHP` and :ref:`Industry demand`). -Solid biomass can be converted to syngas if the option is enabled in the `config file `_. In this case the model will enable the technology BioSNG both with and without the option for carbon capture (see `Technology-data repository `_). +Solid biomass can be converted to syngas if the option is enabled in the `config file `__. In this case the model will enable the technology BioSNG both with and without the option for carbon capture (see `Technology-data repository `__). -Liquefaction of solid biomass `can be enabled `_ allowing the model to convert it into liquid hydrocarbons that can replace conventional oil products. This technology also comes with and without carbon capture (see `Technology-data repository `_). +Liquefaction of solid biomass `can be enabled `__ allowing the model to convert it into liquid hydrocarbons that can replace conventional oil products. This technology also comes with and without carbon capture (see `Technology-data repository `__). *Transport of solid biomass* -The transport of solid biomass can either be assumed unlimited between countries or it can be associated with a country specific cost per MWh/km. In the config file these options are toggled `here `_. If the option is off, use of solid biomass is transport. If it is turned on, a biomass transport network will be `created `_ between all nodes. This network resembles road transport of biomass and the cost of transportation is a variable cost which is proportional to distance and a country specific cost per MWh/km. The latter is `estimated `_ from the country specific costs per ton/km used in the publication `“The JRC-EU-TIMES model. Bioenergy potentials for EU and neighbouring countries” `_. +The transport of solid biomass can either be assumed unlimited between countries or it can be associated with a country specific cost per MWh/km. In the config file these options are toggled `here `__. If the option is off, use of solid biomass is transport. If it is turned on, a biomass transport network will be `created `__ between all nodes. This network resembles road transport of biomass and the cost of transportation is a variable cost which is proportional to distance and a country specific cost per MWh/km. The latter is `estimated `__ from the country specific costs per ton/km used in the publication `“The JRC-EU-TIMES model. Bioenergy potentials for EU and neighbouring countries” `__. *Biogas transport and use* -Biogas will be aggregated into a common European resources if a gas network is not modelled explicitly, i.e., the `gas_network `_ option is set to false. If, on the other hand, a gas network is included, the biogas potential will be associated with each node of origin. +Biogas will be aggregated into a common European resources if a gas network is not modelled explicitly, i.e., the `gas_network `__ option is set to false. If, on the other hand, a gas network is included, the biogas potential will be associated with each node of origin. The model can only use biogas by first upgrading it to natural gas quality [see :ref:`Methane supply`] (bio methane) which is fed into the general gas network. .. _Oil-based products demand: @@ -338,7 +338,7 @@ Oil-based products can be either of fossil origin or synthetically produced by c 𝑛CO+(2𝑛+1)H_2 → C_{n}H_{2n + 2} +𝑛H_2O -with costs as included from the `technology-data repository `_. The waste heat from the Fischer-Tropsch process is supplied to `district heating networks `_. The share of fossil and synthetic oil is an optimisation result depending on the techno-economic assumptions. +with costs as included from the `technology-data repository `__. The waste heat from the Fischer-Tropsch process is supplied to `district heating networks `__. The share of fossil and synthetic oil is an optimisation result depending on the techno-economic assumptions. *Oil-based transport* @@ -361,24 +361,24 @@ The Subsection overview below provides a general description of the modelling ap Greenhouse gas emissions associated with industry can be classified into energy-related and process-related emissions. Today, fossil fuels are used for process heat energy in the chemicals industry, but also as a non-energy feedstock for chemicals like ammonia ( :math:`NH_3`), ethylene ( :math:`C_2H_4`) and methanol ( :math:`CH_3OH`). Energy-related emissions can be curbed by using low-emission energy sources. The only option to reduce process-related emissions is by using an alternative manufacturing process or by assuming a certain rate of recycling so that a lower amount of virgin material is needed. -The overarching modelling procedure can be described as follows. First, the energy demands and process emissions for every unit of material output are estimated based on data from the `JRC-IDEES database `_ and the fuel and process switching described in the subsequent sections. Second, the 2050 energy demands and process emissions are calculated using the per-unit-of-material ratios based on the industry transformations and the `country-level material production in 2015 `_, assuming constant material demand. +The overarching modelling procedure can be described as follows. First, the energy demands and process emissions for every unit of material output are estimated based on data from the `JRC-IDEES database `__ and the fuel and process switching described in the subsequent sections. Second, the 2050 energy demands and process emissions are calculated using the per-unit-of-material ratios based on the industry transformations and the `country-level material production in 2015 `__, assuming constant material demand. -Missing or too coarsely aggregated data in the JRC-IDEES database is supplemented with additional datasets: `Eurostat energy balances `_, `United States `_, `Geological Survey `_ for ammonia production, `DECHEMA `_ for methanol and chlorine, and `national statistics from Switzerland `_. +Missing or too coarsely aggregated data in the JRC-IDEES database is supplemented with additional datasets: `Eurostat energy balances `__, `United States `__, `Geological Survey `__ for ammonia production, `DECHEMA `__ for methanol and chlorine, and `national statistics from Switzerland `__. Where there are fossil and electrified alternatives for the same process (e.g. in glass manufacture or drying), we assume that the process is completely electrified. Current electricity demands (lighting, air compressors, motor drives, fans, pumps) will remain electric. Processes that require temperatures below 500 °C are supplied with solid biomass, since we assume that residues and wastes are not suitable for high-temperature applications. We see solid biomass use primarily in the pulp and paper industry, where it is already widespread, and in food, beverages and tobacco, where it replaces natural gas. Industries which require high temperatures (above 500 °C), such as metals, chemicals and non-metallic minerals are either electrified where suitable processes already exist, or the heat is provided with synthetic methane. Hydrogen for high-temperature process heat is not part of the model currently. -Where process heat is required, our approach depends on the necessary temperature. For example, due to the high share of high-temperature process heat demand (see `Naegler et al. `_ and `Rehfeldt el al. `_), we disregard geothermal and solar thermal energy as sources for process heat since they cannot attain high-temperature heat. +Where process heat is required, our approach depends on the necessary temperature. For example, due to the high share of high-temperature process heat demand (see `Naegler et al. `__ and `Rehfeldt el al. `__), we disregard geothermal and solar thermal energy as sources for process heat since they cannot attain high-temperature heat. -The following figure shows the final consumption of energy and non-energy feedstocks in industry today in comparison to the scenario in 2050 assumed in `Neumann et al `_. +The following figure shows the final consumption of energy and non-energy feedstocks in industry today in comparison to the scenario in 2050 assumed in `Neumann et al `__. .. image:: ../graphics/fec_industry_today_tomorrow.png The following figure shows the process emissions in industry today (top bar) and in 2050 without -carbon capture (bottom bar) assumed in `Neumann et al `_. +carbon capture (bottom bar) assumed in `Neumann et al `__. @@ -386,7 +386,7 @@ carbon capture (bottom bar) assumed in `Neumann et al `_, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries. +Inside each country the industrial demand is then distributed using the `Hotmaps Industrial Database `__, which is illustrated in the figure below. This open database includes georeferenced industrial sites of energy-intensive industry sectors in EU28, including cement, basic chemicals, glass, iron and steel, non-ferrous metals, non-metallic minerals, paper, and refineries subsectors. The use of this spatial dataset enables the calculation of regional and process-specific energy demands. This approach assumes that there will be no significant migration of energy-intensive industries. .. image:: ../graphics/hotmaps.png @@ -395,7 +395,7 @@ Inside each country the industrial demand is then distributed using the `Hotmaps **Iron and Steel** -Two alternative routes are used today to manufacture steel in Europe. The primary route (integrated steelworks) represents 60% of steel production, while the secondary route (electric arc furnaces, EAF), represents the other 40% `(Lechtenböhmer et. al) `_. +Two alternative routes are used today to manufacture steel in Europe. The primary route (integrated steelworks) represents 60% of steel production, while the secondary route (electric arc furnaces, EAF), represents the other 40% `(Lechtenböhmer et. al) `__. The primary route uses blast furnaces in which coke is used to reduce iron ore into molten iron, which is then converted into steel: @@ -415,9 +415,9 @@ The primary route uses blast furnaces in which coke is used to reduce iron ore i FeO + CO \xrightarrow{} Fe + CO_2 -The primary route of steelmaking implies large process emissions of 0.22 t :math:`_{CO_2}` /t of steel, amounting to 7% of global greenhouse gas emissions `(Vogl et. al) `_. +The primary route of steelmaking implies large process emissions of 0.22 t :math:`_{CO_2}` /t of steel, amounting to 7% of global greenhouse gas emissions `(Vogl et. al) `__. -In the secondary route, electric arc furnaces are used to melt scrap metal. This limits the :math:`CO_2` emissions to the burning of graphite electrodes `(Friedrichsen et. al) `_, and reduces process emissions to 0.03 t :math:`_{CO_2}` /t of steel. +In the secondary route, electric arc furnaces are used to melt scrap metal. This limits the :math:`CO_2` emissions to the burning of graphite electrodes `(Friedrichsen et. al) `__, and reduces process emissions to 0.03 t :math:`_{CO_2}` /t of steel. We assume that the primary route can be replaced by a third route in 2050, using direct reduced iron (DRI) and subsequent processing in an EAF. @@ -433,10 +433,10 @@ We assume that the primary route can be replaced by a third route in 2050, using FeO + H_2 \xrightarrow{} Fe + H_2O -This circumvents the process emissions associated with the use of coke. For hydrogen- based DRI, we assume energy requirements of 1.7 MWh :math:`_{H_2}` /t steel `(Vogl et. al) `_ and 0.322 MWh :math:`_{el}`/t steel `(HYBRIT 2016) `_. +This circumvents the process emissions associated with the use of coke. For hydrogen- based DRI, we assume energy requirements of 1.7 MWh :math:`_{H_2}` /t steel `(Vogl et. al) `__ and 0.322 MWh :math:`_{el}`/t steel `(HYBRIT 2016) `__. -The share of steel produced via the primary route is exogenously set in the `config file `_. The share of steel obtained via hydrogen-based DRI plus EAF is also set exogenously in the `config file `_. The remaining share is manufactured through the secondary route using scrap metal in EAF. Bioenergy as alternative to coke in blast furnaces is not considered in the model (`Mandova et.al `_, `Suopajärvi et.al `_). +The share of steel produced via the primary route is exogenously set in the `config file `__. The share of steel obtained via hydrogen-based DRI plus EAF is also set exogenously in the `config file `__. The remaining share is manufactured through the secondary route using scrap metal in EAF. Bioenergy as alternative to coke in blast furnaces is not considered in the model (`Mandova et.al `__, `Suopajärvi et.al `__). For the remaining subprocesses in this sector, the following transformations are assumed. Methane is used as energy source for the smelting process. Activities associated with furnaces, refining and rolling, and product finishing are electrified assuming the current efficiency values for these cases. These transformations result in changes in process emissions as outlined in the process emissions figure presented in the industry overview section (see :ref:`Overview`). @@ -446,28 +446,28 @@ For the remaining subprocesses in this sector, the following transformations are The chemicals industry includes a wide range of diverse industries, including the production of basic organic compounds (olefins, alcohols, aromatics), basic inorganic compounds (ammonia, chlorine), polymers (plastics), and end-user products (cosmetics, pharmaceutics). -The chemicals industry consumes large amounts of fossil-fuel based feedstocks (see `Levi et. al `_), which can also be produced from renewables as outlined for hydrogen (see :ref:`Hydrogen supply`), for methane (see :ref:`Methane supply`), and for oil-based products (see :ref:`Oil-based products supply`). The ratio between synthetic and fossil-based fuels used in the industry is an endogenous result of the optimisation. +The chemicals industry consumes large amounts of fossil-fuel based feedstocks (see `Levi et. al `__), which can also be produced from renewables as outlined for hydrogen (see :ref:`Hydrogen supply`), for methane (see :ref:`Methane supply`), and for oil-based products (see :ref:`Oil-based products supply`). The ratio between synthetic and fossil-based fuels used in the industry is an endogenous result of the optimisation. -The basic chemicals consumption data from the `JRC IDEES `_ database comprises high- value chemicals (ethylene, propylene and BTX), chlorine, methanol and ammonia. However, it is necessary to separate out these chemicals because their current and future production routes are different. +The basic chemicals consumption data from the `JRC IDEES `__ database comprises high- value chemicals (ethylene, propylene and BTX), chlorine, methanol and ammonia. However, it is necessary to separate out these chemicals because their current and future production routes are different. -Statistics for the production of ammonia, which is commonly used as a fertilizer, are taken from the `USGS `_ for every country. Ammonia can be made from hydrogen and nitrogen using the Haber-Bosch process. +Statistics for the production of ammonia, which is commonly used as a fertilizer, are taken from the `USGS `__ for every country. Ammonia can be made from hydrogen and nitrogen using the Haber-Bosch process. .. math:: N_2 + 3H_2 \xrightarrow{} 2NH_3 -The Haber-Bosch process is not explicitly represented in the model, such that demand for ammonia enters the model as a demand for hydrogen ( 6.5 MWh :math:`_{H_2}` / t :math:`_{NH_3}` ) and electricity ( 1.17 MWh :math:`_{el}` /t :math:`_{NH_3}` ) (see `Wang et. al `_). Today, natural gas dominates in Europe as the source for the hydrogen used in the Haber-Bosch process, but the model can choose among the various hydrogen supply options described in the hydrogen section (see :ref:`Hydrogen supply`) +The Haber-Bosch process is not explicitly represented in the model, such that demand for ammonia enters the model as a demand for hydrogen ( 6.5 MWh :math:`_{H_2}` / t :math:`_{NH_3}` ) and electricity ( 1.17 MWh :math:`_{el}` /t :math:`_{NH_3}` ) (see `Wang et. al `__). Today, natural gas dominates in Europe as the source for the hydrogen used in the Haber-Bosch process, but the model can choose among the various hydrogen supply options described in the hydrogen section (see :ref:`Hydrogen supply`) -The total production and specific energy consumption of chlorine and methanol is taken from a `DECHEMA report `_. According to this source, the production of chlorine amounts to 9.58 MtCl/a, which is assumed to require electricity at 3.6 MWh :math:`_{el}`/t of chlorine and yield hydrogen at 0.937 MWh :math:`_{H_2}`/t of chlorine in the chloralkali process. The production of methanol adds up to 1.5 MtMeOH/a, requiring electricity at 0.167 MWh :math:`_{el}`/t of methanol and methane at 10.25 MWh :math:`_{CH_4}`/t of methanol. +The total production and specific energy consumption of chlorine and methanol is taken from a `DECHEMA report `__. According to this source, the production of chlorine amounts to 9.58 MtCl/a, which is assumed to require electricity at 3.6 MWh :math:`_{el}`/t of chlorine and yield hydrogen at 0.937 MWh :math:`_{H_2}`/t of chlorine in the chloralkali process. The production of methanol adds up to 1.5 MtMeOH/a, requiring electricity at 0.167 MWh :math:`_{el}`/t of methanol and methane at 10.25 MWh :math:`_{CH_4}`/t of methanol. -The production of ammonia, methanol, and chlorine production is deducted from the JRC IDEES basic chemicals, leaving the production totals of high-value chemicals. For this, we assume that the liquid hydrocarbon feedstock comes from synthetic or fossil- origin naphtha (14 MWh :math:`_{naphtha}`/t of HVC, similar to `Lechtenböhmer et al `_), ignoring the methanol-to-olefin route. Furthermore, we assume the following transformations of the energy-consuming processes in the production of plastics: the final energy consumption in steam processing is converted to methane since requires temperature above 500 °C (4.1 MWh :math:`_{CH_4}` /t of HVC, see `Rehfeldt et al. `_); and the remaining processes are electrified using the current efficiency of microwave for high-enthalpy heat processing, electric furnaces, electric process cooling and electric generic processes (2.85 MWh :math:`_{el}`/t of HVC). +The production of ammonia, methanol, and chlorine production is deducted from the JRC IDEES basic chemicals, leaving the production totals of high-value chemicals. For this, we assume that the liquid hydrocarbon feedstock comes from synthetic or fossil- origin naphtha (14 MWh :math:`_{naphtha}`/t of HVC, similar to `Lechtenböhmer et al `__), ignoring the methanol-to-olefin route. Furthermore, we assume the following transformations of the energy-consuming processes in the production of plastics: the final energy consumption in steam processing is converted to methane since requires temperature above 500 °C (4.1 MWh :math:`_{CH_4}` /t of HVC, see `Rehfeldt et al. `__); and the remaining processes are electrified using the current efficiency of microwave for high-enthalpy heat processing, electric furnaces, electric process cooling and electric generic processes (2.85 MWh :math:`_{el}`/t of HVC). The process emissions from feedstock in the chemical industry are as high as 0.369 t :math:`_{CO_2}`/t of ethylene equivalent. We consider process emissions for all the material output, which is a conservative approach since it assumes that all plastic-embedded :math:`CO_2` will eventually be released into the atmosphere. However, plastic disposal in landfilling will avoid, or at least delay, associated :math:`CO_2` emissions. -Circular economy practices drastically reduce the amount of primary feedstock needed for the production of plastics in the model (see `Kullmann et al. `_, `Meys et al. (2021) `_, `Meys et al. (2020) `_, `Gu et al. `_) and consequently, also the energy demands and level of process emission. The percentage of plastics that are assumed to be mechanically recycled can be selected in the `config file `_, as well as -the percentage that is chemically recycled, see `config file `_ The energy consumption for those recycling processes are respectively 0.547 MWh :math:`_{el}`/t of HVC (as indicated in the `config file `_) (`Meys et al. (2020) `_), and 6.9 MWh :math:`_{el}`/t of HVC (as indicated in the `config file `_) based on pyrolysis and electric steam cracking (see `Materials Economics `_ report). +Circular economy practices drastically reduce the amount of primary feedstock needed for the production of plastics in the model (see `Kullmann et al. `__, `Meys et al. (2021) `__, `Meys et al. (2020) `__, `Gu et al. `__) and consequently, also the energy demands and level of process emission. The percentage of plastics that are assumed to be mechanically recycled can be selected in the `config file `__, as well as +the percentage that is chemically recycled, see `config file `__ The energy consumption for those recycling processes are respectively 0.547 MWh :math:`_{el}`/t of HVC (as indicated in the `config file `__) (`Meys et al. (2020) `__), and 6.9 MWh :math:`_{el}`/t of HVC (as indicated in the `config file `__) based on pyrolysis and electric steam cracking (see `Materials Economics `__ report). **Non-metallic Mineral Products** @@ -476,7 +476,7 @@ This subsector includes the manufacturing of cement, ceramics, and glass. *Cement* -Cement is used in construction to make concrete. The production of cement involves high energy consumption and large process emissions. The calcination of limestone to chemically reactive calcium oxide, also known as lime, involves process emissions of 0.54 t :math:`_{CO_2}` /t cement (see `Akhtar et al. `_. +Cement is used in construction to make concrete. The production of cement involves high energy consumption and large process emissions. The calcination of limestone to chemically reactive calcium oxide, also known as lime, involves process emissions of 0.54 t :math:`_{CO_2}` /t cement (see `Akhtar et al. `__. .. math:: @@ -487,16 +487,16 @@ Additionally, :math:`CO_2` is emitted from the combustion of fossil fuels to pro Cement process emissions can be captured assuming a capture rate of 90%. Whether emissions are captured is decided by the model taking into account the capital costs of carbon capture modules. The electricity and heat demand of process emission carbon capture is currently ignored. For net-zero emission scenarios, the remaining process emissions need to be compensated by negative emissions. -With the exception of electricity demand and biomass demand for low-temperature heat (0.06 MWh/t and 0.2 MWh/t), the final energy consumption of this subsector is assumed to be supplied by methane (0.52 MWh/t), which is capable of delivering the required high-temperature heat. This implies a switch from burning solid fuels to burning gas which will require adjustments of the `kilns <10.1109/CITCON.2013.6525276>`_. The share of fossil vs. synthetic methane consumed is a result of the optimisation +With the exception of electricity demand and biomass demand for low-temperature heat (0.06 MWh/t and 0.2 MWh/t), the final energy consumption of this subsector is assumed to be supplied by methane (0.52 MWh/t), which is capable of delivering the required high-temperature heat. This implies a switch from burning solid fuels to burning gas which will require adjustments of the `kilns <10.1109/CITCON.2013.6525276>`__. The share of fossil vs. synthetic methane consumed is a result of the optimisation *Ceramics* -The ceramics sector is assumed to be fully electrified based on the current efficiency of already electrified processes which include microwave drying and sintering of raw materials, electric kilns for primary production processes, electric furnaces for the `product finishing `_. In total, the final electricity consumption is 0.44 MWh/t of ceramic. The manufacturing of ceramics includes process emissions of 0.03 t :math:`_{CO_2}`/t of ceramic. For a detailed overview of the ceramics industry sector see `Furszyfer Del Rio et al `_. +The ceramics sector is assumed to be fully electrified based on the current efficiency of already electrified processes which include microwave drying and sintering of raw materials, electric kilns for primary production processes, electric furnaces for the `product finishing `__. In total, the final electricity consumption is 0.44 MWh/t of ceramic. The manufacturing of ceramics includes process emissions of 0.03 t :math:`_{CO_2}`/t of ceramic. For a detailed overview of the ceramics industry sector see `Furszyfer Del Rio et al `__. *Glass* -The production of glass is assumed to be fully electrified based on the current efficiency of electric melting tanks and electric annealing which adds up to an electricity demand of 2.07 MWh :math:`_{el}`/t of `glass `_. The manufacturing of glass incurs process emissions of 0.1 t :math:`_{CO_2}`/t of glass. Potential efficiency improvements, which according to `Lechtenböhmer et al `_ could reduce energy demands to 0.85 MW :math:`_{el}`/t of glass, have not been considered. For a detailed overview of the glass industry sector see `Furszyfer Del Rio et al `_. +The production of glass is assumed to be fully electrified based on the current efficiency of electric melting tanks and electric annealing which adds up to an electricity demand of 2.07 MWh :math:`_{el}`/t of `glass `__. The manufacturing of glass incurs process emissions of 0.1 t :math:`_{CO_2}`/t of glass. Potential efficiency improvements, which according to `Lechtenböhmer et al `__ could reduce energy demands to 0.85 MW :math:`_{el}`/t of glass, have not been considered. For a detailed overview of the glass industry sector see `Furszyfer Del Rio et al `__. **Non-ferrous Metals** @@ -511,75 +511,75 @@ The primary route involves two energy-intensive processes: the production of alu 2Al_2O_3 +3C \xrightarrow{} 4Al+3CO_2 -The primary route requires high-enthalpy heat (2.3 MWh/t) to produce alumina which is supplied by methane and causes process emissions of 1.5 t :math:`_{CO_2}`/t aluminium. According to `Friedrichsen et al. `_, inert anodes might become commercially available by 2030 that would eliminate the process emissions, but they are not included in the model. Assuming all subprocesses are electrified, the primary route requires 15.4 MWh :math:`_{el}`/t of aluminium. +The primary route requires high-enthalpy heat (2.3 MWh/t) to produce alumina which is supplied by methane and causes process emissions of 1.5 t :math:`_{CO_2}`/t aluminium. According to `Friedrichsen et al. `__, inert anodes might become commercially available by 2030 that would eliminate the process emissions, but they are not included in the model. Assuming all subprocesses are electrified, the primary route requires 15.4 MWh :math:`_{el}`/t of aluminium. -In the secondary route, scrap aluminium is remelted. The energy demand for this process is only 10% of the primary route and there are no associated process emissions. Assuming all subprocesses are electrified, the secondary route requires 1.7 MWh/t of aluminium. The share of aliminum manufactured by the primary and secondary route can be selected in the `config file `_] +In the secondary route, scrap aluminium is remelted. The energy demand for this process is only 10% of the primary route and there are no associated process emissions. Assuming all subprocesses are electrified, the secondary route requires 1.7 MWh/t of aluminium. The share of aliminum manufactured by the primary and secondary route can be selected in the `config file `__] For the other non-ferrous metals, we assume the electrification of the entire manufacturing process with an average electricity demand of 3.2 MWh :math:`_{el}`/t lead equivalent. **Other Industry Subsectors** -The remaining industry subsectors include (a) pulp, paper, printing, (b) food, beverages, tobacco, (c) textiles and leather, (d) machinery equipment, (e) transport equipment, (f) wood and wood products, (g) others. Low- and mid-temperature process heat in these industries is assumed to be `supplied by biomass `_ while the remaining processes are electrified. None of the subsectors involve process emissions. +The remaining industry subsectors include (a) pulp, paper, printing, (b) food, beverages, tobacco, (c) textiles and leather, (d) machinery equipment, (e) transport equipment, (f) wood and wood products, (g) others. Low- and mid-temperature process heat in these industries is assumed to be `supplied by biomass `__ while the remaining processes are electrified. None of the subsectors involve process emissions. Agriculture demand ========================= -Energy demands for the agriculture, forestry and fishing sector per country are taken from the `JRC-IDEES database `_. Missing countries are filled with `Eurostat data `_. Agricultural energy demands are split into electricity (lighting, ventilation, specific electricity uses, electric pumping devices), heat (specific heat uses, low enthalpy heat), and machinery oil (motor drives, farming machine drives, diesel-fueled pumping devices). Heat demand is assigned at “services rural heat” buses. Time series for demands are assumed to be constant and distributed inside countries by population. +Energy demands for the agriculture, forestry and fishing sector per country are taken from the `JRC-IDEES database `__. Missing countries are filled with `Eurostat data `__. Agricultural energy demands are split into electricity (lighting, ventilation, specific electricity uses, electric pumping devices), heat (specific heat uses, low enthalpy heat), and machinery oil (motor drives, farming machine drives, diesel-fueled pumping devices). Heat demand is assigned at “services rural heat” buses. Time series for demands are assumed to be constant and distributed inside countries by population. .. _Transportation: Transportation ========================= -Annual energy demands for land transport, aviation and shipping for every country are retrieved from `JRC-IDEES data set `_. Below, the details of how each of these categories are treated is explained. +Annual energy demands for land transport, aviation and shipping for every country are retrieved from `JRC-IDEES data set `__. Below, the details of how each of these categories are treated is explained. .. _Land transport: **Land transport** -Both road and rail transport is combined as `land transport demand `_ although electrified rail transport is excluded because that demand is included in the current electricity demand. +Both road and rail transport is combined as `land transport demand `__ although electrified rail transport is excluded because that demand is included in the current electricity demand. -The most important settings for land transport are the exogenously fixed fuel mix (an option enabling the endogeous optimization of transport electrification is planned but not yet implemented). In the `config file `_, the share of battery electric vehicles (BEV) and hydrogen fuel cell vehicles (FCEV) can be set. The remaining percentage will be treated as internal combustion engines (ICE) that consume oil products. +The most important settings for land transport are the exogenously fixed fuel mix (an option enabling the endogeous optimization of transport electrification is planned but not yet implemented). In the `config file `__, the share of battery electric vehicles (BEV) and hydrogen fuel cell vehicles (FCEV) can be set. The remaining percentage will be treated as internal combustion engines (ICE) that consume oil products. *Battery Electric vehicles (BEV)* -For the electrified land transport, country-specific factors are computed by comparing the `current car final energy consumption per km in `_ (average for Europe 0.7 kWh/km) to the 0.18 kWh/km value assumed for battery-to-wheels efficiency in EVs. The characteristic `weekly profile `_ provided by the German Federal Highway Research Institute (BASt) is used to obtain hourly time series for European countries taking into account the corresponding local times. Furthermore, a temperature dependence is included in the time series to account for heating/cooling demand in transport. For temperatures `below `_/`above `_ certain threshold values, e.g. 15 °C/20 °C, `temperature coefficients `_ of typically 0.98%/°C and 0.63%/°C are assumed, based on the `paper `_. +For the electrified land transport, country-specific factors are computed by comparing the `current car final energy consumption per km in `__ (average for Europe 0.7 kWh/km) to the 0.18 kWh/km value assumed for battery-to-wheels efficiency in EVs. The characteristic `weekly profile `__ provided by the German Federal Highway Research Institute (BASt) is used to obtain hourly time series for European countries taking into account the corresponding local times. Furthermore, a temperature dependence is included in the time series to account for heating/cooling demand in transport. For temperatures `below `__/`above `__ certain threshold values, e.g. 15 °C/20 °C, `temperature coefficients `__ of typically 0.98%/°C and 0.63%/°C are assumed, based on the `paper `__. -For BEVs the user can define the `storage energy capacity `_, `charging power capacity `_, and `charging efficiency `_. +For BEVs the user can define the `storage energy capacity `__, `charging power capacity `__, and `charging efficiency `__. -For BEV, smart charging is an option. A `certain share `_ of the BEV fleet can shift their charging time. The BEV state of charge is forced to be higher than a `set percentage `_, e.g. 75%, every day at a `specified hour `_, e.g., 7 am, to ensure that the batteries are sufficiently charged for peak usage in the morning and they not behave as seasonal storage. They also have the option to participate in vehicle-to-grid (V2G) services to facilitate system operation if that `is enabled `_. +For BEV, smart charging is an option. A `certain share `__ of the BEV fleet can shift their charging time. The BEV state of charge is forced to be higher than a `set percentage `__, e.g. 75%, every day at a `specified hour `__, e.g., 7 am, to ensure that the batteries are sufficiently charged for peak usage in the morning and they not behave as seasonal storage. They also have the option to participate in vehicle-to-grid (V2G) services to facilitate system operation if that `is enabled `__. The battery cost of BEV is not included in the model since it is assumed that BEV owners buy them to primarily satisfy their mobility needs. *Hydrogen fuel cell vehicles (FCEV)* The share of all land transport that is specified to be be FCEV will be converted to a demand for hydrogen (see :ref:`Hydrogen supply`) using the `FCEV efficiency -`_. +`__. FCEVs are typically used to simulate demand for transport that is hard to electrify directly, e.g. heavy construction machinery. But it may also be used to investigate a more widespread adoption of the technology. *Internal combustion engine vehicles (ICE)* All land transport that is not specified to be either BEV or FCEV will be treated as conventional ICEs. The transport demand is converted to a demand for oil products (see :ref:`Oil-based products supply`) using the `ICE efficiency -`_. +`__. .. _Aviation: **Aviation** -The `demand for aviation `_ includes international and domestic use. It is modelled as an oil demand since aviation consumes kerosene. This can be produced synthetically or have fossil-origin (see :ref:`Oil-based products supply`). +The `demand for aviation `__ includes international and domestic use. It is modelled as an oil demand since aviation consumes kerosene. This can be produced synthetically or have fossil-origin (see :ref:`Oil-based products supply`). .. _Shipping: **Shipping** -Shipping energy demand is covered by a combination of oil and hydrogen. Other fuel options, like methanol or ammonia, are currently not included in PyPSA-Eur-Sec. The share of shipping that is assumed to be supplied by hydrogen can be selected in the `config file `_. +Shipping energy demand is covered by a combination of oil and hydrogen. Other fuel options, like methanol or ammonia, are currently not included in PyPSA-Eur-Sec. The share of shipping that is assumed to be supplied by hydrogen can be selected in the `config file `__. -To estimate the `hydrogen demand `_, the average fuel efficiency of the fleet is used in combination with the efficiency of the fuel cell defined in the technology-data repository. The average fuel efficiency is set in the `config file `_. +To estimate the `hydrogen demand `__, the average fuel efficiency of the fleet is used in combination with the efficiency of the fuel cell defined in the technology-data repository. The average fuel efficiency is set in the `config file `__. The consumed hydrogen comes from the general hydrogen bus where it can be produced by SMR, SMR+CC or electrolysers (see :ref:`Hydrogen supply`). The fraction that is not converted into hydrogen use oil products, i.e. is connected to the general oil bus. -The energy demand for liquefaction of the hydrogen used for shipping can be `included `_. If this option is selected, liquifaction will happen at the `node where the shipping demand occurs `_. +The energy demand for liquefaction of the hydrogen used for shipping can be `included `__. If this option is selected, liquifaction will happen at the `node where the shipping demand occurs `__. .. _Carbon dioxide capture, usage and sequestration (CCU/S): @@ -600,12 +600,12 @@ For the following point source emissions, carbon capture is applicable: • CHP plants using biomass or methane -• `Coal power plants `_. +• `Coal power plants `__. -Point source emissions are captured assuming a capture rate, e.g. 90%, which can be specified in the `config file `_. The electricity and heat demand of process emission carbon capture +Point source emissions are captured assuming a capture rate, e.g. 90%, which can be specified in the `config file `__. The electricity and heat demand of process emission carbon capture is currently ignored. -DAC (if `included `_) includes the adsorption phase where electricity and heat consumptionsare required to assist the adsorption process and regenerate the adsorbent. It also includes the drying and compression of :math:`CO_2` prior to storage which consumes electricity and rejects heat. +DAC (if `included `__) includes the adsorption phase where electricity and heat consumptionsare required to assist the adsorption process and regenerate the adsorbent. It also includes the drying and compression of :math:`CO_2` prior to storage which consumes electricity and rejects heat. *Carbon dioxide usage* @@ -614,8 +614,8 @@ naphtha). If captured carbon is used, the :math:`CO_2` emissions of the syntheti *Carbon dioxide sequestration* -Captured :math:`CO_2` can also be sequestered underground up to an annual sequestration limit of 200 Mt :math:`_{CO_2}`/a. This limit can be chosen in the `config file `_. As stored carbon dioxide is modelled as a single node for Europe, :math:`CO_2` transport constraints are neglected. Since :math:`CO_2` sequestration is an immature technology, the cost assumption is defined in the `config file `_. +Captured :math:`CO_2` can also be sequestered underground up to an annual sequestration limit of 200 Mt :math:`_{CO_2}`/a. This limit can be chosen in the `config file `__. As stored carbon dioxide is modelled as a single node for Europe, :math:`CO_2` transport constraints are neglected. Since :math:`CO_2` sequestration is an immature technology, the cost assumption is defined in the `config file `__. *Carbon dioxide transport* -Carbon dioxide can be modelled as a single node for Europe (in this case, :math:`CO_2` transport constraints are neglected). A network for modelling the transport of :math:`CO_2` among the different nodes can also be created if selected in the `config file `_. +Carbon dioxide can be modelled as a single node for Europe (in this case, :math:`CO_2` transport constraints are neglected). A network for modelling the transport of :math:`CO_2` among the different nodes can also be created if selected in the `config file `__. diff --git a/doc/support.rst b/doc/support.rst index 1a3e6d089..fc86a9983 100644 --- a/doc/support.rst +++ b/doc/support.rst @@ -7,8 +7,8 @@ Support ####################### -* In case of code-related **questions**, please post on `stack overflow `_. -* For non-programming related and more general questions please refer to the `mailing list `_. -* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the `discord server `_. -* For **bugs and feature requests**, please use the `issue tracker `_. -* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `_. For further information on how to contribute, please refer to :ref:`contributing`. +* In case of code-related **questions**, please post on `stack overflow `__. +* For non-programming related and more general questions please refer to the `mailing list `__. +* To **discuss** with other PyPSA users, organise projects, share news, and get in touch with the community you can use the `discord server `__. +* For **bugs and feature requests**, please use the `issue tracker `__. +* We strongly welcome anyone interested in providing **contributions** to this project. If you have any ideas, suggestions or encounter problems, feel invited to file issues or make pull requests on `Github `__. For further information on how to contribute, please refer to :ref:`contributing`. diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 4f272292d..93bb8e54e 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -323,4 +323,4 @@ Jupyter Notebooks). n = pypsa.Network("results/networks/elec_s_6_ec_lcopt_Co2L-24H.nc") -For inspiration, read the `examples section in the PyPSA documentation `_. +For inspiration, read the `examples section in the PyPSA documentation `__. diff --git a/doc/validation.rst b/doc/validation.rst index e538717ce..afe7a7f37 100644 --- a/doc/validation.rst +++ b/doc/validation.rst @@ -9,7 +9,7 @@ Validation The PyPSA-Eur model workflow provides a built-in mechanism for validation. This allows users to contrast the outcomes of network optimization against the historical behaviour of the European power system. The snakemake rule ``validate_elec_networks`` enables this by generating comparative figures that encapsulate key data points such as dispatch carrier, cross-border flows, and market prices per price zone. -These comparisons utilize data from the 2019 ENTSO-E Transparency Platform. To enable this, an ENTSO-E API key must be inserted into the ``config.yaml`` file. Detailed steps for this process can be found in the user guide `here `_. +These comparisons utilize data from the 2019 ENTSO-E Transparency Platform. To enable this, an ENTSO-E API key must be inserted into the ``config.yaml`` file. Detailed steps for this process can be found in the user guide `here `__. Once the API key is set, the validation workflow can be triggered by running the following command: diff --git a/doc/wildcards.rst b/doc/wildcards.rst index f86ff3115..1fd1646f8 100644 --- a/doc/wildcards.rst +++ b/doc/wildcards.rst @@ -17,7 +17,7 @@ what data to retrieve and what files to produce. .. note:: Detailed explanations of how wildcards work in ``snakemake`` can be found in the - `relevant section of the documentation `_. + `relevant section of the documentation `__. .. _cutout_wc: From 8273b7854027bfba8c961856820dbdfc783689ec Mon Sep 17 00:00:00 2001 From: toniseibold Date: Thu, 11 Apr 2024 16:32:08 +0200 Subject: [PATCH 25/57] file is not used --- data/hydrogen_salt_cavern_potentials.csv | 31 ------------------------ 1 file changed, 31 deletions(-) delete mode 100644 data/hydrogen_salt_cavern_potentials.csv diff --git a/data/hydrogen_salt_cavern_potentials.csv b/data/hydrogen_salt_cavern_potentials.csv deleted file mode 100644 index c1168266c..000000000 --- a/data/hydrogen_salt_cavern_potentials.csv +++ /dev/null @@ -1,31 +0,0 @@ -ct,TWh -AT, -BA, -BE, -BG, -CH, -CZ, -DE,4500 -DK,700 -EE, -ES,350 -FI, -FR, -GB,1050 -GR,120 -HR, -HU, -IE, -IT, -LT, -LU, -LV, -NL,150 -NO, -PL,120 -PT,400 -RO, -RS, -SE, -SI, -SK, From 4d4b8ea2666e141977f7b168452561e37a79f16a Mon Sep 17 00:00:00 2001 From: cpschau Date: Thu, 11 Apr 2024 13:45:02 +0200 Subject: [PATCH 26/57] n.madd instead of pd.concat --- scripts/base_network.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 528f04bfa..706c112e5 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -704,8 +704,14 @@ def _set_shapes(n, country_shapes, offshore_shapes): country_shapes["type"] = "country" offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) offshore_shapes["type"] = "offshore" - all_shapes = pd.concat([country_shapes, offshore_shapes]) - n.shapes = pd.concat([n.shapes, all_shapes], ignore_index=True) + all_shapes = pd.concat([country_shapes, offshore_shapes], ignore_index=True) + n.madd( + "Shape", + all_shapes.index, + geometry=all_shapes.geometry, + idx=all_shapes.idx, + type=all_shapes.type, + ) def base_network( From f2db3c63270b94aabd1cf2d65f16ede0e446b5a2 Mon Sep 17 00:00:00 2001 From: cpschau Date: Thu, 11 Apr 2024 16:26:16 +0200 Subject: [PATCH 27/57] add AC & DC lines --- scripts/base_network.py | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/scripts/base_network.py b/scripts/base_network.py index 706c112e5..007505022 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -705,6 +705,10 @@ def _set_shapes(n, country_shapes, offshore_shapes): offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) offshore_shapes["type"] = "offshore" all_shapes = pd.concat([country_shapes, offshore_shapes], ignore_index=True) +<<<<<<< HEAD +======= + +>>>>>>> dce7d57a (add AC & DC lines) n.madd( "Shape", all_shapes.index, @@ -712,6 +716,69 @@ def _set_shapes(n, country_shapes, offshore_shapes): idx=all_shapes.idx, type=all_shapes.type, ) +<<<<<<< HEAD +======= + + # Write the AC and DC line shapes to the network.shapes component + start_index = n.shapes.index.astype(int).max() + 1 + index_AC = pd.RangeIndex(start=start_index, stop=start_index + len(n.lines)) + geo_AC = gpd.GeoSeries( + n.lines.geometry.apply(shapely.wkt.loads).fillna( + n.lines[["bus0", "bus1"]].apply( + lambda x: LineString( + [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] + ), + axis=1, + ) + ) + ) + geo_AC.index = index_AC + + n.madd( + "Shape", + index_AC, + geometry=geo_AC.values, + idx=n.lines.index, + component="Line", + type=n.lines.carrier.values, + ) + + if n.links.empty: + return + start_index = n.shapes.index.astype(int).max() + 1 + index_DC = pd.RangeIndex(start=start_index, stop=start_index + len(n.links)) + if "geometry" in n.links.columns: + geo_DC = gpd.GeoSeries( + n.links.geometry.apply(shapely.wkt.loads).fillna( + n.links[["bus0", "bus1"]].apply( + lambda x: LineString( + [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] + ), + axis=1, + ) + ) + ) + else: + geo_DC = gpd.GeoSeries( + n.links[["bus0", "bus1"]].apply( + lambda x: LineString( + [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] + ), + axis=1, + ) + ) + + geo_DC = gpd.GeoSeries(geo_DC) + + n.madd( + "Shape", + index_DC, + geometry=geo_DC.values, + idx=n.links.index, + component="Link", + type=n.links.carrier.values, + ) +>>>>>>> dce7d57a (add AC & DC lines) def base_network( From c68e3420c0e400398e1cb1fe5cc5295148d157e1 Mon Sep 17 00:00:00 2001 From: cpschau Date: Thu, 11 Apr 2024 16:42:19 +0200 Subject: [PATCH 28/57] clean-up after rebase --- scripts/base_network.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 007505022..916241a97 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -705,19 +705,13 @@ def _set_shapes(n, country_shapes, offshore_shapes): offshore_shapes = gpd.read_file(offshore_shapes).rename(columns={"name": "idx"}) offshore_shapes["type"] = "offshore" all_shapes = pd.concat([country_shapes, offshore_shapes], ignore_index=True) -<<<<<<< HEAD -======= - ->>>>>>> dce7d57a (add AC & DC lines) n.madd( "Shape", all_shapes.index, geometry=all_shapes.geometry, idx=all_shapes.idx, - type=all_shapes.type, + type=all_shapes["type"], ) -<<<<<<< HEAD -======= # Write the AC and DC line shapes to the network.shapes component start_index = n.shapes.index.astype(int).max() + 1 @@ -778,7 +772,6 @@ def _set_shapes(n, country_shapes, offshore_shapes): component="Link", type=n.links.carrier.values, ) ->>>>>>> dce7d57a (add AC & DC lines) def base_network( From 35713e82b4b25604bc5c5afd7a99bcbd656f5660 Mon Sep 17 00:00:00 2001 From: toniseibold Date: Thu, 11 Apr 2024 16:44:18 +0200 Subject: [PATCH 29/57] Remove heat_load_profile_DK_AdamJensen.csv and license comment --- data/heat_load_profile_DK_AdamJensen.csv | 25 ------------------------ doc/configtables/licenses-sector.csv | 1 - 2 files changed, 26 deletions(-) delete mode 100644 data/heat_load_profile_DK_AdamJensen.csv diff --git a/data/heat_load_profile_DK_AdamJensen.csv b/data/heat_load_profile_DK_AdamJensen.csv deleted file mode 100644 index cb417bde7..000000000 --- a/data/heat_load_profile_DK_AdamJensen.csv +++ /dev/null @@ -1,25 +0,0 @@ -hour,weekday,weekend -0,0.9181438689,0.9421512708 -1,0.9172359071,0.9400891069 -2,0.9269464481,0.9461062015 -3,0.9415047932,0.9535084941 -4,0.9656299507,0.9651094993 -5,1.0221166443,0.9834676747 -6,1.1553090493,1.0124171051 -7,1.2093411031,1.0446615927 -8,1.1470295942,1.088203419 -9,1.0877191341,1.1110334576 -10,1.0418327372,1.0926752822 -11,1.0062977133,1.055488209 -12,0.9837030359,1.0251266112 -13,0.9667570278,0.9990015154 -14,0.9548320932,0.9782897278 -15,0.9509232061,0.9698167237 -16,0.9636973319,0.974288587 -17,0.9799372563,0.9886456216 -18,1.0046501848,1.0084159643 -19,1.0079452419,1.0171243296 -20,0.9860566481,0.9994722379 -21,0.9705228074,0.982761591 -22,0.9586485819,0.9698167237 -23,0.9335023778,0.9515079292 diff --git a/doc/configtables/licenses-sector.csv b/doc/configtables/licenses-sector.csv index 7f20b5a6f..d1660596b 100644 --- a/doc/configtables/licenses-sector.csv +++ b/doc/configtables/licenses-sector.csv @@ -8,7 +8,6 @@ Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa. Swiss energy statistics from Swiss Federal Office of Energy,switzerland-sfoe/,unknown,http://www.bfe.admin.ch/themen/00526/00541/00542/02167/index.html?dossier_id=02169 BASt emobility statistics,emobility/,unknown,http://www.bast.de/DE/Verkehrstechnik/Fachthemen/v2-verkehrszaehlung/Stundenwerte.html?nn=626916 BDEW heating profile,heat_load_profile_BDEW.csv,unknown,https://github.com/oemof/demandlib -heating profiles for Aarhus,heat_load_profile_DK_AdamJensen.csv,unknown,Adam Jensen MA thesis at Aarhus University co2 budgets,co2_budget.csv,CC BY 4.0,https://arxiv.org/abs/2004.11009 existing heating potentials,existing_infrastructure/existing_heating_raw.csv,unknown,https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en IRENA existing VRE capacities,existing_infrastructure/{solar|onwind|offwind}_capcity_IRENA.csv,unknown,https://www.irena.org/Statistics/Download-Data From 1867e510e26d22790d1efd1c50078c213082a71a Mon Sep 17 00:00:00 2001 From: cpschau Date: Thu, 11 Apr 2024 17:14:17 +0200 Subject: [PATCH 30/57] remove redundant line --- scripts/base_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 916241a97..663e995f3 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -762,8 +762,6 @@ def _set_shapes(n, country_shapes, offshore_shapes): ) ) - geo_DC = gpd.GeoSeries(geo_DC) - n.madd( "Shape", index_DC, From 7dd1d8fb59065f4b6782caf46f5261f5530f625a Mon Sep 17 00:00:00 2001 From: toniseibold Date: Thu, 11 Apr 2024 17:46:44 +0200 Subject: [PATCH 31/57] delete biomass JRC from license file --- doc/configtables/licenses-sector.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/configtables/licenses-sector.csv b/doc/configtables/licenses-sector.csv index d1660596b..a44e0a5d5 100644 --- a/doc/configtables/licenses-sector.csv +++ b/doc/configtables/licenses-sector.csv @@ -1,7 +1,6 @@ description,file/folder,licence,source JRC IDEES database,jrc-idees-2015/,CC BY 4.0,https://ec.europa.eu/jrc/en/potencia/jrc-idees urban/rural fraction,urban_percent.csv,unknown,unknown -JRC biomass potentials,biomass/,unknown,https://doi.org/10.2790/39014 JRC ENSPRESO biomass potentials,remote,CC BY 4.0,https://data.jrc.ec.europa.eu/dataset/74ed5a04-7d74-4807-9eab-b94774309d9f EEA emission statistics,eea/UNFCCC_v23.csv,EEA standard re-use policy,https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 Eurostat Energy Balances,eurostat-energy_balances-*/,Eurostat,https://ec.europa.eu/eurostat/web/energy/data/energy-balances From 038575dfc0c3dd9139c2253aa6cfa2c3ff596d40 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 11 Apr 2024 19:14:45 +0200 Subject: [PATCH 32/57] skip heat bus for CHPs in places where no central heating --- scripts/add_existing_baseyear.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f445efe34..f10280991 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -362,13 +362,20 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ) else: key = "central solid biomass CHP" + central_heat = n.buses.query( + "carrier == 'urban central heat'" + ).location.unique() + heat_buses = new_capacity.index.map( + lambda i: i + " urban central heat" if i in central_heat else "" + ) + n.madd( "Link", new_capacity.index, suffix=name_suffix, bus0=spatial.biomass.df.loc[new_capacity.index]["nodes"].values, bus1=new_capacity.index, - bus2=new_capacity.index + " urban central heat", + bus2=heat_buses, carrier=generator, p_nom=new_capacity / costs.at[key, "efficiency"], capital_cost=costs.at[key, "fixed"] From 3b31b99dd2d2f48c30eda741b6b21c07b0895cef Mon Sep 17 00:00:00 2001 From: toniseibold Date: Fri, 12 Apr 2024 10:01:09 +0200 Subject: [PATCH 33/57] removing the output file eia_hydro_stats.csv --- rules/build_electricity.smk | 1 - scripts/build_hydro_profile.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f84a99691..65e241896 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -360,7 +360,6 @@ rule build_hydro_profile: + ".nc", output: profile=resources("profile_hydro.nc"), - eia_hydro=resources("eia_hydro_stats.csv"), log: logs("build_hydro_profile.log"), resources: diff --git a/scripts/build_hydro_profile.py b/scripts/build_hydro_profile.py index c4e9701e3..cd51ce907 100644 --- a/scripts/build_hydro_profile.py +++ b/scripts/build_hydro_profile.py @@ -198,8 +198,6 @@ def approximate_missing_eia_stats(eia_stats, runoff_fn, countries): fn = snakemake.input.era5_runoff eia_stats = approximate_missing_eia_stats(eia_stats, fn, countries) - eia_stats.to_csv(snakemake.output.eia_hydro) - contained_years = pd.date_range(freq="YE", **snakemake.params.snapshots).year norm_year = config_hydro.get("eia_norm_year") missing_years = contained_years.difference(eia_stats.index) From 98d2c4dd911b6d0157ea11691b057f32837a1760 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 11 Apr 2024 10:54:25 +0200 Subject: [PATCH 34/57] Fix p_nom_min of renewables generators for myopic approach and add check of existing capacities in add_land_use_constraint_m --- doc/release_notes.rst | 2 ++ scripts/solve_network.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d42b149f8..d9c692449 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -181,6 +181,8 @@ Upcoming Release * Fix custom busmap read in `cluster_network`. +* Fix p_nom_min of renewables generators for myopic approach and add check of existing capacities in `add_land_use_constraint_m`. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0f6725c70..b8819a19b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -159,6 +159,9 @@ def _add_land_use_constraint_m(n, planning_horizons, config): current_horizon = snakemake.wildcards.planning_horizons for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: + extendable_i = (n.generators.carrier == carrier) & n.generators.p_nom_extendable + n.generators.loc[extendable_i, "p_nom_min"] = 0 + existing = n.generators.loc[n.generators.carrier == carrier, "p_nom"] ind = list( {i.split(sep=" ")[0] + " " + i.split(sep=" ")[1] for i in existing.index} @@ -180,6 +183,19 @@ def _add_land_use_constraint_m(n, planning_horizons, config): sel_p_year ].rename(lambda x: x[:-4] + current_horizon) + # check if existing capacities are larger than technical potential + existing_large = n.generators[ + n.generators["p_nom_min"] > n.generators["p_nom_max"] + ].index + if len(existing_large): + logger.warning( + f"Existing capacities larger than technical potential for {existing_large},\ + adjust technical potential to existing capacities" + ) + n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ + existing_large, "p_nom_min" + ] + n.generators.p_nom_max.clip(lower=0, inplace=True) From 7a0ca73f3a1f7e2feae8446fb9815fd6b2a13b93 Mon Sep 17 00:00:00 2001 From: Sermisha Date: Sun, 14 Apr 2024 14:24:01 +0530 Subject: [PATCH 35/57] Update to BAU constraint formulation in function add_BAU_constraints Constraint formulation was not taking into account the ordering of the carriers --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0f6725c70..b1463b353 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -543,7 +543,7 @@ def add_BAU_constraints(n, config): ext_carrier_i = xr.DataArray(ext_i.carrier.rename_axis("Generator-ext")) lhs = p_nom.groupby(ext_carrier_i).sum() index = mincaps.index.intersection(lhs.indexes["carrier"]) - rhs = mincaps[index].rename_axis("carrier") + rhs = mincaps[lhs.indexes["carrier"]].rename_axis("carrier") n.model.add_constraints(lhs >= rhs, name="bau_mincaps") From 4429711b15bbacdd3ef0cad49d09b189b65fa67a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:51:09 +0000 Subject: [PATCH 36/57] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 24.3.0 → 24.4.0](https://github.com/psf/black-pre-commit-mirror/compare/24.3.0...24.4.0) - [github.com/snakemake/snakefmt: v0.10.0 → v0.10.1](https://github.com/snakemake/snakefmt/compare/v0.10.0...v0.10.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 055b90804..a3aab9eda 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 + rev: 24.4.0 hooks: # Format Python files - id: black @@ -74,7 +74,7 @@ repos: # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt - rev: v0.10.0 + rev: v0.10.1 hooks: - id: snakefmt From 15345ff1cfae4026a69c070d02f5893642f6b0f2 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 11 Apr 2024 19:14:45 +0200 Subject: [PATCH 37/57] skip heat bus for CHPs in places where no central heating --- scripts/add_existing_baseyear.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f445efe34..f10280991 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -362,13 +362,20 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ) else: key = "central solid biomass CHP" + central_heat = n.buses.query( + "carrier == 'urban central heat'" + ).location.unique() + heat_buses = new_capacity.index.map( + lambda i: i + " urban central heat" if i in central_heat else "" + ) + n.madd( "Link", new_capacity.index, suffix=name_suffix, bus0=spatial.biomass.df.loc[new_capacity.index]["nodes"].values, bus1=new_capacity.index, - bus2=new_capacity.index + " urban central heat", + bus2=heat_buses, carrier=generator, p_nom=new_capacity / costs.at[key, "efficiency"], capital_cost=costs.at[key, "fixed"] From 380884e9ab16337c6f1fbc9d2bc489376066f2c9 Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 14 Mar 2024 13:44:15 +0100 Subject: [PATCH 38/57] group existing capacities to the earlier grouping_year for consistency with optimized capacities --- config/config.default.yaml | 2 +- doc/release_notes.rst | 1 + scripts/add_existing_baseyear.py | 20 ++++++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index d438c51f7..0d40bafc9 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -361,7 +361,7 @@ solar_thermal: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] - grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 + grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020] # heat grouping years >= baseyear will be ignored threshold_capacity: 10 default_heating_lifetime: 20 conventional_carriers: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d9c692449..d9075ac71 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,6 +9,7 @@ Release Notes Upcoming Release ================ +* Group existing capacities to the earlier grouping_year for consistency with optimized capacities * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential. diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f10280991..c10e9deb2 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -190,8 +190,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas df_agg.drop(phased_out, inplace=True) df_agg["grouping_year"] = np.take( - grouping_years, np.digitize(df_agg.DateIn, grouping_years, right=True) - ) + grouping_years[::-1], + np.digitize(df_agg.DateIn, grouping_years[::-1])) # calculate (adjusted) remaining lifetime before phase-out (+1 because assuming # phase out date at the end of the year) @@ -451,12 +451,16 @@ def add_heating_capacities_installed_before_baseyear( else: efficiency = costs.at[costs_name, "efficiency"] - for i, grouping_year in enumerate(grouping_years): - if int(grouping_year) + default_lifetime <= int(baseyear): - continue - - # installation is assumed to be linear for the past default_lifetime years - ratio = (int(grouping_year) - int(grouping_years[i - 1])) / default_lifetime + valid_grouping_years = pd.Series( + [int(gy) for gy in grouping_years if + int(baseyear) - default_lifetime <= int(gy) < int(baseyear)]) + # Installation is assumed to be linear for the past + _intervals = pd.concat( + [valid_grouping_years[1:], pd.Series(baseyear)], + ignore_index=True) - valid_grouping_years + ratios = _intervals / _intervals.sum() + + for ratio, grouping_year in zip(ratios, valid_grouping_years): n.madd( "Link", From 27f2bd77fb6f8cea85e149bd3a918f9c0af2688a Mon Sep 17 00:00:00 2001 From: Michael Lindner Date: Thu, 14 Mar 2024 16:13:40 +0100 Subject: [PATCH 39/57] add more grouping years to avoid errors --- config/config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 0d40bafc9..42132f226 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -360,7 +360,7 @@ solar_thermal: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities existing_capacities: - grouping_years_power: [1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] + grouping_years_power: [1895, 1920, 1950, 1955, 1960, 1965, 1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020] # heat grouping years >= baseyear will be ignored threshold_capacity: 10 default_heating_lifetime: 20 From 17d44d7f64d76878cadcabf09064ac53a2306dd5 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 08:54:15 +0200 Subject: [PATCH 40/57] add check if minimum grouping year< min build year --- scripts/add_existing_baseyear.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c10e9deb2..ddd409c1f 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -189,6 +189,15 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas phased_out = df_agg[df_agg["DateOut"] < baseyear].index df_agg.drop(phased_out, inplace=True) + older_assets = (df_agg.DateIn Date: Thu, 11 Apr 2024 09:26:30 +0200 Subject: [PATCH 41/57] adjust valid_grouping_years change sign, split up conditions for better readibility --- scripts/add_existing_baseyear.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index ddd409c1f..285776cc4 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -461,14 +461,22 @@ def add_heating_capacities_installed_before_baseyear( efficiency = costs.at[costs_name, "efficiency"] valid_grouping_years = pd.Series( - [int(gy) for gy in grouping_years if - int(baseyear) - default_lifetime <= int(gy) < int(baseyear)]) + [ + int(grouping_year) + for grouping_year in grouping_years + if int(grouping_year) + default_lifetime > int(baseyear) + and int(grouping_year) < int(baseyear) + ] + ) # Installation is assumed to be linear for the past - _intervals = pd.concat( - [valid_grouping_years[1:], pd.Series(baseyear)], - ignore_index=True) - valid_grouping_years + _intervals = ( + pd.concat( + [valid_grouping_years[1:], pd.Series(baseyear)], ignore_index=True + ) + - valid_grouping_years + ) ratios = _intervals / _intervals.sum() - + for ratio, grouping_year in zip(ratios, valid_grouping_years): n.madd( From 6f09a16b3a8f1f10c1808f3f42d7cdb5f0ff394c Mon Sep 17 00:00:00 2001 From: Micha Date: Thu, 11 Apr 2024 09:37:36 +0200 Subject: [PATCH 42/57] Add release note for serendipitous bugfix --- doc/release_notes.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d9075ac71..660fa3573 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -9,7 +9,9 @@ Release Notes Upcoming Release ================ -* Group existing capacities to the earlier grouping_year for consistency with optimized capacities +* Group existing capacities to the earlier grouping_year for consistency with optimized capacities. + +* bugfix: installed heating capacities were 5% lower than existing heating capacities * Include gas and oil fields and saline aquifers in estimation of CO2 sequestration potential. From dd874e0ed8ae9ef2a99e1bdf51d093f7adeef958 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:52:17 +0200 Subject: [PATCH 43/57] rename _intervals to _years, shorten definition --- scripts/add_existing_baseyear.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 285776cc4..db2c27e42 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -468,14 +468,12 @@ def add_heating_capacities_installed_before_baseyear( and int(grouping_year) < int(baseyear) ] ) + + # get number of years of each interval + _years = (valid_grouping_years.diff().shift(-1) + .fillna(baseyear-valid_grouping_years.iloc[-1])) # Installation is assumed to be linear for the past - _intervals = ( - pd.concat( - [valid_grouping_years[1:], pd.Series(baseyear)], ignore_index=True - ) - - valid_grouping_years - ) - ratios = _intervals / _intervals.sum() + ratios = _years / _years.sum() for ratio, grouping_year in zip(ratios, valid_grouping_years): From ec96a73baf745ecff3c10e418ae5dcde8ac19958 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:54:33 +0200 Subject: [PATCH 44/57] define active assets consitently --- scripts/add_brownfield.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 7e49031b7..16b4e0870 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -40,8 +40,8 @@ def add_brownfield(n, n_p, year): # CO2 or global EU values since these are already in n n_p.mremove(c.name, c.df.index[c.df.lifetime == np.inf]) - # remove assets whose build_year + lifetime < year - n_p.mremove(c.name, c.df.index[c.df.build_year + c.df.lifetime < year]) + # remove assets whose build_year + lifetime <= year + n_p.mremove(c.name, c.df.index[c.df.build_year + c.df.lifetime <= year]) # remove assets if their optimized nominal capacity is lower than a threshold # since CHP heat Link is proportional to CHP electric Link, make sure threshold is compatible From da15ab3ac25ea6c85dc60d100e239cd666c81bbd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 12:21:43 +0000 Subject: [PATCH 45/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index db2c27e42..e75fe5a2a 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -189,18 +189,20 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas phased_out = df_agg[df_agg["DateOut"] < baseyear].index df_agg.drop(phased_out, inplace=True) - older_assets = (df_agg.DateIn Date: Tue, 16 Apr 2024 10:24:16 +0200 Subject: [PATCH 46/57] no addition of straight lines --- scripts/base_network.py | 58 ----------------------------------------- 1 file changed, 58 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 663e995f3..6727a7245 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -713,64 +713,6 @@ def _set_shapes(n, country_shapes, offshore_shapes): type=all_shapes["type"], ) - # Write the AC and DC line shapes to the network.shapes component - start_index = n.shapes.index.astype(int).max() + 1 - index_AC = pd.RangeIndex(start=start_index, stop=start_index + len(n.lines)) - geo_AC = gpd.GeoSeries( - n.lines.geometry.apply(shapely.wkt.loads).fillna( - n.lines[["bus0", "bus1"]].apply( - lambda x: LineString( - [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] - ), - axis=1, - ) - ) - ) - geo_AC.index = index_AC - - n.madd( - "Shape", - index_AC, - geometry=geo_AC.values, - idx=n.lines.index, - component="Line", - type=n.lines.carrier.values, - ) - - if n.links.empty: - return - start_index = n.shapes.index.astype(int).max() + 1 - index_DC = pd.RangeIndex(start=start_index, stop=start_index + len(n.links)) - if "geometry" in n.links.columns: - geo_DC = gpd.GeoSeries( - n.links.geometry.apply(shapely.wkt.loads).fillna( - n.links[["bus0", "bus1"]].apply( - lambda x: LineString( - [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] - ), - axis=1, - ) - ) - ) - else: - geo_DC = gpd.GeoSeries( - n.links[["bus0", "bus1"]].apply( - lambda x: LineString( - [n.buses.loc[x[0], ["x", "y"]], n.buses.loc[x[1], ["x", "y"]]] - ), - axis=1, - ) - ) - - n.madd( - "Shape", - index_DC, - geometry=geo_DC.values, - idx=n.links.index, - component="Link", - type=n.links.carrier.values, - ) - def base_network( eg_buses, From 6aac8c90b4c221e11a9d44df7ddcf91d7765a3f6 Mon Sep 17 00:00:00 2001 From: cpschau Date: Tue, 16 Apr 2024 10:30:39 +0200 Subject: [PATCH 47/57] added release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d42b149f8..4441f439b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -181,6 +181,8 @@ Upcoming Release * Fix custom busmap read in `cluster_network`. +* Added shapes to .nc file for different stages of the network object in `base_network`, `build_bus_regions`, and `cluster_network`. + PyPSA-Eur 0.10.0 (19th February 2024) ===================================== From 6ea8d52a81085d3396d831289bd5c3b0f50c602c Mon Sep 17 00:00:00 2001 From: cpschau Date: Tue, 16 Apr 2024 10:43:07 +0200 Subject: [PATCH 48/57] no directory change before mock_snakemake --- scripts/base_network.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 6727a7245..d96a7e541 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -781,12 +781,9 @@ def base_network( if __name__ == "__main__": if "snakemake" not in globals(): - import os from _helpers import mock_snakemake - os.chdir(os.path.dirname(os.path.abspath(__file__))) - snakemake = mock_snakemake("base_network") configure_logging(snakemake) set_scenario_config(snakemake) From feb564506a1c11ee7bb0b1c20be29a0e8027656f Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 16 Apr 2024 15:41:01 +0200 Subject: [PATCH 49/57] remove connection_costs as snakemake output from simplify_network --- rules/build_electricity.smk | 1 - scripts/simplify_network.py | 12 ++---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 65e241896..ed341d2f8 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -487,7 +487,6 @@ rule simplify_network: regions_onshore=resources("regions_onshore_elec_s{simpl}.geojson"), regions_offshore=resources("regions_offshore_elec_s{simpl}.geojson"), busmap=resources("busmap_elec_s{simpl}.csv"), - connection_costs=resources("connection_costs_s{simpl}.csv"), log: logs("simplify_network/elec_s{simpl}.log"), benchmark: diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 24df7312e..d74fcdb53 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -207,7 +207,7 @@ def _compute_connection_costs_to_bus( return connection_costs_to_bus -def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output): +def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus): connection_costs = {} for tech in connection_costs_to_bus: tech_b = n.generators.carrier == tech @@ -228,14 +228,12 @@ def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, out ) ) connection_costs[tech] = costs - pd.DataFrame(connection_costs).to_csv(output.connection_costs) def _aggregate_and_move_components( n, busmap, connection_costs_to_bus, - output, aggregate_one_ports={"Load", "StorageUnit"}, aggregation_strategies=dict(), exclude_carriers=None, @@ -248,7 +246,7 @@ def replace_components(n, c, df, pnl): if not df.empty: import_series_from_dataframe(n, df, c, attr) - _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output) + _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus) generator_strategies = aggregation_strategies["generators"] @@ -281,7 +279,6 @@ def simplify_links( length_factor, p_max_pu, exclude_carriers, - output, aggregation_strategies=dict(), ): ## Complex multi-node links are folded into end-points @@ -406,7 +403,6 @@ def split_links(nodes): n, busmap, connection_costs_to_bus, - output, aggregation_strategies=aggregation_strategies, exclude_carriers=exclude_carriers, ) @@ -419,7 +415,6 @@ def remove_stubs( renewable_carriers, length_factor, simplify_network, - output, aggregation_strategies=dict(), ): logger.info("Removing stubs") @@ -436,7 +431,6 @@ def remove_stubs( n, busmap, connection_costs_to_bus, - output, aggregation_strategies=aggregation_strategies, exclude_carriers=simplify_network["exclude_carriers"], ) @@ -556,7 +550,6 @@ def cluster( params.length_factor, params.p_max_pu, params.simplify_network["exclude_carriers"], - snakemake.output, params.aggregation_strategies, ) @@ -569,7 +562,6 @@ def cluster( params.renewable_carriers, params.length_factor, params.simplify_network, - snakemake.output, aggregation_strategies=params.aggregation_strategies, ) busmaps.append(stub_map) From 48f199f93671338c0f8a5e159804e4e8c752cbbd Mon Sep 17 00:00:00 2001 From: martacki Date: Tue, 16 Apr 2024 15:46:36 +0200 Subject: [PATCH 50/57] remove UA from countries list for synthetic load data --- scripts/build_electricity_demand.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_electricity_demand.py b/scripts/build_electricity_demand.py index 9174672d5..7615bbc63 100755 --- a/scripts/build_electricity_demand.py +++ b/scripts/build_electricity_demand.py @@ -311,6 +311,8 @@ def manual_adjustment(load, fn_load, countries): logger.info("Supplement missing data with synthetic data.") fn = snakemake.input.synthetic synthetic_load = pd.read_csv(fn, index_col=0, parse_dates=True) + # "UA" does not appear in synthetic load data + countries = list(set(countries) - set(["UA"])) synthetic_load = synthetic_load.loc[snapshots, countries] load = load.combine_first(synthetic_load) From c505ea174e7b2272db98fdeefae2540062ea7ad5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 24 Apr 2024 16:23:43 +0200 Subject: [PATCH 51/57] doc: update solver requirements [skip ci] --- config/config.default.yaml | 2 +- doc/configtables/solving.csv | 2 +- doc/installation.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 42132f226..0db6dc037 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -770,7 +770,7 @@ solving: solver_options: highs-default: - # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver + # refer to https://ergo-code.github.io/HiGHS/dev/options/definitions/ threads: 4 solver: "ipm" run_crossover: "off" diff --git a/doc/configtables/solving.csv b/doc/configtables/solving.csv index 7189399b3..4d2451953 100644 --- a/doc/configtables/solving.csv +++ b/doc/configtables/solving.csv @@ -20,7 +20,7 @@ constraints ,,, -- BAU,bool,"{'true','false'}",Add a per-``carrier`` minimal overall capacity; i.e. at least ``40GW`` of ``OCGT`` in Europe; configured in ``electricity: BAU_mincapacities`` -- SAFE,bool,"{'true','false'}",Add a capacity reserve margin of a certain fraction above the peak demand to which renewable generators and storage do *not* contribute. Ignores network. solver,,, --- name,--,"One of {'gurobi', 'cplex', 'cbc', 'glpk', 'ipopt'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow. +-- name,--,"One of {'gurobi', 'cplex', 'highs', 'cbc', 'glpk'}; potentially more possible",Solver to use for optimisation problems in the workflow; e.g. clustering and linear optimal power flow. -- options,--,Key listed under ``solver_options``.,Link to specific parameter settings. solver_options,,dict,Dictionaries with solver-specific parameter settings. mem,MB,int,Estimated maximum memory requirement for solving networks. diff --git a/doc/installation.rst b/doc/installation.rst index dd5c912e6..45404e1f9 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -62,7 +62,7 @@ PyPSA is known to work with the free software - `HiGHS `__ - `Cbc `__ - `GLPK `__ (`WinGLKP `__) -- `Ipopt `__ +- `SCIP `__ and the non-free, commercial software (for some of which free academic licenses are available) @@ -80,7 +80,7 @@ Nevertheless, you can still use open-source solvers for smaller problems. .. note:: The rules :mod:`cluster_network` and :mod:`simplify_network` solve a mixed-integer quadratic optimisation problem for clustering. - The open-source solvers HiGHS, Cbc and GlPK cannot handle this. A fallback to SCIP is implemented in this case. + The open-source solvers HiGHS, Cbc and GlPK cannot handle this. A fallback to SCIP is implemented in this case, which is included in the standard environment specifications. For an open-source solver setup install in your ``conda`` environment on OSX/Linux. To install the default solver Gurobi, run .. code:: bash From 29f6c9507f01c5a9cca7f8435c6260143f60dd6f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:36:31 +0000 Subject: [PATCH 52/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ed341d2f8..c074bdbd2 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -433,7 +433,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -478,7 +478,7 @@ rule simplify_network: input: network=resources("networks/elec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), @@ -526,7 +526,7 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), @@ -555,7 +555,7 @@ rule add_extra_components: input: network=resources("networks/elec_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), @@ -590,7 +590,7 @@ rule prepare_network: input: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: From 884d308d8b1de46988db184c94443299d1935293 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 25 Apr 2024 17:45:51 +0200 Subject: [PATCH 53/57] Bump powerplantmatching version Needed in order to get renewable capacities from pm.IRENA newer than 2020; updated to 2023 in version 0.5.13 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index ee1d1605a..8f4258052 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -25,7 +25,7 @@ dependencies: - yaml - pytables - lxml -- powerplantmatching>=0.5.5,!=0.5.9 +- powerplantmatching>=0.5.13 - numpy - pandas>=2.1 - geopandas>=0.11.0 From a4a2af5404abf0a5452d68a22986b7cb7d0943ee Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 25 Apr 2024 18:05:41 +0200 Subject: [PATCH 54/57] Fix double space in existing capacities The `name_suffix` variable already includes a space after the bus name. --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index e75fe5a2a..0ae231252 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -310,7 +310,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas n.madd( "Generator", new_capacity.index, - suffix=" " + name_suffix, + suffix=name_suffix, bus=new_capacity.index, carrier=generator, p_nom=new_capacity, From 105cadaf7b83a1a549482c3f39f33e2285a0894c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:08:45 +0000 Subject: [PATCH 55/57] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ed341d2f8..c074bdbd2 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -433,7 +433,7 @@ rule add_electricity: else resources("networks/base.nc") ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions=resources("regions_onshore.geojson"), powerplants=resources("powerplants.csv"), @@ -478,7 +478,7 @@ rule simplify_network: input: network=resources("networks/elec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), regions_onshore=resources("regions_onshore.geojson"), regions_offshore=resources("regions_offshore.geojson"), @@ -526,7 +526,7 @@ rule cluster_network: else [] ), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: network=resources("networks/elec_s{simpl}_{clusters}.nc"), @@ -555,7 +555,7 @@ rule add_extra_components: input: network=resources("networks/elec_s{simpl}_{clusters}.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), output: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), @@ -590,7 +590,7 @@ rule prepare_network: input: resources("networks/elec_s{simpl}_{clusters}_ec.nc"), tech_costs=lambda w: resources( - f"costs_{config_provider('costs', 'year')(w)}.csv" + f"costs_{config_provider('costs', 'year') (w)}.csv" ), co2_price=lambda w: resources("co2_price.csv") if "Ept" in w.opts else [], output: From edc1d4991a0c84330a6390d043880447a6db7a40 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 26 Apr 2024 09:35:54 +0200 Subject: [PATCH 56/57] reenable windows ci with snakemake 8.11 --- .github/workflows/ci.yaml | 2 +- Snakefile | 2 +- envs/environment.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7aabf0e6c..c17c0425f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: os: - ubuntu-latest - macos-latest -# - windows-latest + - windows-latest inhouse: - stable - master diff --git a/Snakefile b/Snakefile index ba93a869f..e69807089 100644 --- a/Snakefile +++ b/Snakefile @@ -8,7 +8,7 @@ from os.path import normpath, exists from shutil import copyfile, move, rmtree from snakemake.utils import min_version -min_version("8.5") +min_version("8.11") from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir diff --git a/envs/environment.yaml b/envs/environment.yaml index ee1d1605a..0a378fdc8 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -20,7 +20,7 @@ dependencies: - openpyxl!=3.1.1 - pycountry - seaborn -- snakemake-minimal>=8.5 +- snakemake-minimal>=8.11 - memory_profiler - yaml - pytables From 162d376ce96753fb09c24d3968b7115ecf29cad9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 19:55:49 +0000 Subject: [PATCH 57/57] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black-pre-commit-mirror: 24.4.0 → 24.4.2](https://github.com/psf/black-pre-commit-mirror/compare/24.4.0...24.4.2) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a3aab9eda..3ffe8d9e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: # Formatting with "black" coding style - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.0 + rev: 24.4.2 hooks: # Format Python files - id: black