From 5c2e3c9fc136b12ea8953fada095d335541d22ca Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 09:58:25 +0200 Subject: [PATCH 1/6] add Shape2Shapes in add_electricit.py --- scripts/add_electricity.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1d32bce18..64cf442ea 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,7 +94,9 @@ import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from vresutils import transfer as vtransfer +from itertools import product +import scipy.sparse as sparse +from shapely.prepared import prep idx = pd.IndexSlice @@ -215,6 +217,20 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) +def Shapes2Shapes(orig, dest): + """ + Adopted from vresutils.transfer.Shapes2Shapes() + """ + orig_prepped = list(map(prep, orig)) + transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) + + for i,j in product(range(len(dest)), range(len(orig))): + if orig_prepped[j].intersects(dest[i]): + area = orig[j].intersection(dest[i]).area + transfer[i,j] = area/dest[i].area + + return transfer + def attach_load(n, regions, load, nuts3_shapes, countries, scaling=1.0): substation_lv_i = n.buses.index[n.buses["substation_lv"]] @@ -232,7 +248,7 @@ def upsample(cntry, group): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = vtransfer.Shapes2Shapes( + transfer = Shapes2Shapes( group, nuts3_cntry.geometry, normed=False ).T.tocsr() gdp_n = pd.Series( From e91068196b9db670ca95d00184e273f7890be61b Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Wed, 10 May 2023 10:02:41 +0200 Subject: [PATCH 2/6] replace vresutils annuity function with add_electricity.py calculate_annuity --- scripts/prepare_sector_network.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1cb7146e4..944c3caa3 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -28,7 +28,7 @@ from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from vresutils.costdata import annuity +from add_electricity import calculate_annuity logger = logging.getLogger(__name__) @@ -742,7 +742,7 @@ def prepare_costs(cost_file, config, nyears): costs = costs.fillna(config["fill_values"]) def annuity_factor(v): - return annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 + return calculate_annuity(v["lifetime"], v["discount rate"]) + v["FOM"] / 100 costs["fixed"] = [ annuity_factor(v) * v["investment"] * nyears for i, v in costs.iterrows() @@ -851,7 +851,7 @@ def add_wave(n, wave_cost_factor): capacity = pd.Series({"Attenuator": 750, "F2HB": 1000, "MultiPA": 600}) # in EUR/MW - annuity_factor = annuity(25, 0.07) + 0.03 + annuity_factor = calculate_annuity(25, 0.07) + 0.03 costs = ( 1e6 * wave_cost_factor From b21965a98600b4fcd3bbf286ac9b5f9bd9032fa6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 08:09:43 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 9 +++++---- scripts/prepare_sector_network.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 64cf442ea..83ec4bb9f 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -85,17 +85,17 @@ """ import logging +from itertools import product import geopandas as gpd import numpy as np import pandas as pd import powerplantmatching as pm import pypsa +import scipy.sparse as sparse import xarray as xr from _helpers import configure_logging, update_p_nom_max from powerplantmatching.export import map_country_bus -from itertools import product -import scipy.sparse as sparse from shapely.prepared import prep idx = pd.IndexSlice @@ -217,6 +217,7 @@ def load_powerplants(ppl_fn): .replace({"carrier": carrier_dict}) ) + def Shapes2Shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() @@ -224,10 +225,10 @@ def Shapes2Shapes(orig, dest): orig_prepped = list(map(prep, orig)) transfer = sparse.lil_matrix((len(dest), len(orig)), dtype=float) - for i,j in product(range(len(dest)), range(len(orig))): + for i, j in product(range(len(dest)), range(len(orig))): if orig_prepped[j].intersects(dest[i]): area = orig[j].intersection(dest[i]).area - transfer[i,j] = area/dest[i].area + transfer[i, j] = area / dest[i].area return transfer diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 944c3caa3..012c97147 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -22,13 +22,13 @@ override_component_attrs, update_config_with_sector_opts, ) +from add_electricity import calculate_annuity from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from pypsa.geo import haversine_pts from pypsa.io import import_components_from_dataframe from scipy.stats import beta -from add_electricity import calculate_annuity logger = logging.getLogger(__name__) From 654f46f8683cf0613db90d38ab1127a9db62ffd4 Mon Sep 17 00:00:00 2001 From: virio-andreyana <114650479+virio-andreyana@users.noreply.github.com> Date: Thu, 11 May 2023 16:58:35 +0200 Subject: [PATCH 4/6] alpha testing complete vresutils removal --- envs/environment.fixed.yaml | 3 +-- envs/environment.yaml | 3 +-- scripts/add_electricity.py | 6 ++---- scripts/solve_network.py | 31 ++++++++++++----------------- scripts/solve_operations_network.py | 30 ++++++++++++---------------- 5 files changed, 30 insertions(+), 43 deletions(-) diff --git a/envs/environment.fixed.yaml b/envs/environment.fixed.yaml index 7d8fcc45e..1ff9313d4 100644 --- a/envs/environment.fixed.yaml +++ b/envs/environment.fixed.yaml @@ -226,7 +226,7 @@ dependencies: - nspr=4.35 - nss=3.88 - numexpr=2.8.3 -- numpy=1.23.5 +- numpy=1.24 - openjdk=17.0.3 - openjpeg=2.5.0 - openpyxl=3.1.0 @@ -378,4 +378,3 @@ dependencies: - highspy==1.5.0.dev0 - pybind11==2.10.3 - tsam==2.2.2 - - vresutils==0.3.1 diff --git a/envs/environment.yaml b/envs/environment.yaml index 0a9891a50..f970c9ba4 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -25,7 +25,7 @@ dependencies: - pytables - lxml - powerplantmatching>=0.5.5 -- numpy<1.24 +- numpy - pandas>=1.4 - geopandas>=0.11.0 - xarray @@ -55,5 +55,4 @@ dependencies: - rasterio!=1.2.10 - pip: - - vresutils>=0.3.1 - tsam>=1.1.0 diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 83ec4bb9f..f910dee44 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -218,7 +218,7 @@ def load_powerplants(ppl_fn): ) -def Shapes2Shapes(orig, dest): +def shapes_to_shapes(orig, dest): """ Adopted from vresutils.transfer.Shapes2Shapes() """ @@ -249,9 +249,7 @@ def upsample(cntry, group): return pd.DataFrame({group.index[0]: l}) else: nuts3_cntry = nuts3.loc[nuts3.country == cntry] - transfer = Shapes2Shapes( - group, nuts3_cntry.geometry, normed=False - ).T.tocsr() + transfer = shapes_to_shapes(group, nuts3_cntry.geometry).T.tocsr() gdp_n = pd.Series( transfer.dot(nuts3_cntry["gdp"].fillna(1.0).values), index=group.index ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ff1c0ccf0..53ac8816f 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -38,7 +38,6 @@ override_component_attrs, update_config_with_sector_opts, ) -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) @@ -667,23 +666,19 @@ def solve_network(n, config, opts="", **kwargs): np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input.keys(): - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n = prepare_network(n, solve_opts, config=snakemake.config) - - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input.keys(): + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) + + n = prepare_network(n, solve_opts, config=snakemake.config) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 25fe0753a..9c2fe3c11 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -46,23 +46,19 @@ np.random.seed(solve_opts.get("seed", 123)) - fn = getattr(snakemake.log, "memory", None) - with memory_logger(filename=fn, interval=30.0) as mem: - if "overrides" in snakemake.input: - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) - else: - n = pypsa.Network(snakemake.input.network) - - n.optimize.fix_optimal_capacities() - n = prepare_network(n, solve_opts, config=snakemake.config) - n = solve_network( - n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + if "overrides" in snakemake.input: + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network( + snakemake.input.network, override_component_attrs=overrides ) + else: + n = pypsa.Network(snakemake.input.network) - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) - n.export_to_netcdf(snakemake.output[0]) + n.optimize.fix_optimal_capacities() + n = prepare_network(n, solve_opts, config=snakemake.config) + n = solve_network( + n, config=snakemake.config, opts=opts, log_fn=snakemake.log.solver + ) - logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) + n.export_to_netcdf(snakemake.output[0]) From 7d6d6d2805f34a6de6d6e1400d376e8970886c4d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 May 2023 14:59:10 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +--- scripts/solve_operations_network.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 53ac8816f..8f4fdf5be 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -668,9 +668,7 @@ def solve_network(n, config, opts="", **kwargs): if "overrides" in snakemake.input.keys(): overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 9c2fe3c11..987302bb3 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -48,9 +48,7 @@ if "overrides" in snakemake.input: overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network( - snakemake.input.network, override_component_attrs=overrides - ) + n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) else: n = pypsa.Network(snakemake.input.network) From e29571535b504c2db3733e877ca0d0c70248b07f Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 12 May 2023 07:11:41 +0200 Subject: [PATCH 6/6] remove references to memory logfile in *.smk, add release note --- doc/release_notes.rst | 2 ++ doc/requirements.txt | 1 - rules/solve_electricity.smk | 4 ---- rules/solve_myopic.smk | 2 -- rules/solve_overnight.smk | 2 -- scripts/solve_operations_network.py | 1 - 6 files changed, 2 insertions(+), 10 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8e6bfdd4c..14f2939d1 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -23,6 +23,8 @@ Upcoming Release hydrogen fuel cell. Add switches for both re-electrification options under ``sector: hydrogen_turbine:`` and ``sector: hydrogen_fuel_cell:``. +* Remove ``vresutils`` dependency. + PyPSA-Eur 0.8.0 (18th March 2023) ================================= diff --git a/doc/requirements.txt b/doc/requirements.txt index 2f08b8d9e..3e760c81f 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -9,7 +9,6 @@ sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 pypsa -vresutils>=0.3.1 powerplantmatching>=0.5.5 atlite>=0.2.9 dask[distributed] diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8ddeca92f..fc70ce421 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,8 +14,6 @@ rule solve_network: ), python=LOGS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_python.log", - memory=LOGS - + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_memory.log", benchmark: BENCHMARKS + "solve_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}" threads: 4 @@ -41,8 +39,6 @@ rule solve_operations_network: ), python=LOGS + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_python.log", - memory=LOGS - + "solve_operations_network/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}_op_memory.log", benchmark: ( BENCHMARKS diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f10d81572..ec4281ffd 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -90,8 +90,6 @@ rule solve_sector_network_myopic: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: 4 resources: mem_mb=config["solving"]["mem"], diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index c2e103e53..f05925b06 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -21,8 +21,6 @@ rule solve_sector_network: + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_solver.log", python=LOGS + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_python.log", - memory=LOGS - + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}_memory.log", threads: config["solving"]["solver"].get("threads", 4) resources: mem_mb=config["solving"]["mem"], diff --git a/scripts/solve_operations_network.py b/scripts/solve_operations_network.py index 987302bb3..27520485c 100644 --- a/scripts/solve_operations_network.py +++ b/scripts/solve_operations_network.py @@ -17,7 +17,6 @@ update_config_with_sector_opts, ) from solve_network import prepare_network, solve_network -from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__)