Linopy integration for Power and Sector-Coupled Modelling

.github/workflows/test.yml

@@ -31,8 +31,8 @@ jobs:
           env_file: envs/linux-pinned.yaml
         - os: macos
           env_file: envs/macos-pinned.yaml
-        # - os: windows
-        #   env_file: envs/windows-pinned.yaml
+        - os: windows
+          env_file: envs/windows-pinned.yaml
 
     defaults:
       run:

Snakefile

@@ -564,6 +564,7 @@ rule add_electricity:
 
 rule simplify_network:
     params:
+        aggregation_strategies=config["cluster_options"]["aggregation_strategies"],
         renewable=config["renewable"],
         geo_crs=config["crs"]["geo_crs"],
         cluster_options=config["cluster_options"],
@@ -606,6 +607,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == True:
 
     rule cluster_network:
         params:
+            aggregation_strategies=config["cluster_options"]["aggregation_strategies"],
             build_shape_options=config["build_shape_options"],
             electricity=config["electricity"],
             costs=config["costs"],
@@ -691,6 +693,7 @@ if config["augmented_line_connection"].get("add_to_snakefile", False) == False:
 
     rule cluster_network:
         params:
+            aggregation_strategies=config["cluster_options"]["aggregation_strategies"],
             build_shape_options=config["build_shape_options"],
             electricity=config["electricity"],
             costs=config["costs"],

config.default.yaml

@@ -99,6 +99,7 @@ cluster_options:
       p_nom_max: sum
       p_nom_min: sum
       p_min_pu: mean
+      p_max_pu: weighted_average
       marginal_cost: mean
       committable: any
       ramp_limit_up: max

doc/release_notes.rst

@@ -142,6 +142,7 @@ PyPSA-Earth 0.4.0
 
 * Add an option to use csv format for custom demand imports. `PR #995 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/995>`__
 
+
 **Minor Changes and bug-fixing**
 
 * Minor bug-fixing to run the cluster wildcard min `PR #1019 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1019>`__

envs/environment.yaml

@@ -12,7 +12,8 @@ dependencies:
 - pip
 - mamba   # esp for windows build
 
-- pypsa>=0.24, <0.25
+- pypsa>=0.25
+# - atlite>=0.2.4  # until https://github.com/PyPSA/atlite/issues/244 is not merged
 - dask
 # currently the packages are being installed with pip
 # need to move back to conda once the issues will be resolved
@@ -28,6 +29,7 @@ dependencies:
 - memory_profiler
 - ruamel.yaml<=0.17.26
 - pytables
+- pyscipopt # added to compy with the quadratic objective requirement of the clustering script
 - lxml
 - numpy
 # starting from 1.3.5 numpoly requires numpy>2.0 which leads to issues
@@ -80,7 +82,6 @@ dependencies:
 
 # Default solver for tests (required for CI)
 - glpk
-- ipopt
 - gurobi
 
 - pip:

envs/linux-pinned.yaml

@@ -92,6 +92,7 @@ dependencies:
 - contourpy=1.3.1
 - country_converter=1.2
 - cpp-expected=1.1.0
+- cppad=20240000.7
 - cycler=0.12.1
 - cyrus-sasl=2.1.27
 - cytoolz=1.0.1
@@ -350,6 +351,7 @@ dependencies:
 - metis=5.1.0
 - minizip=4.0.7
 - mistune=3.0.2
+- mpfr=4.2.1
 - mpg123=1.32.9
 - msgpack-python=1.1.0
 - multipledispatch=0.6.0
@@ -432,13 +434,14 @@ dependencies:
 - pydoe2=1.3.0
 - pygments=2.18.0
 - pyogrio=0.10.0
-- pyomo=6.8.2
+- pyomo=6.6.1
 - pyparsing=3.2.0
 - pyppmd=1.1.0
 - pyproj=3.7.0
-- pypsa=0.24.0
+- pypsa=0.28.0
 - pyqt=5.15.9
 - pyqt5-sip=12.12.2
+- pyscipopt=5.2.1
 - pyshp=2.3.1
 - pysocks=1.7.1
 - pytables=3.10.1
@@ -474,6 +477,7 @@ dependencies:
 - ruamel.yaml.clib=0.2.8
 - s2n=1.5.10
 - scikit-learn=1.6.0
+- scip=9.2.0
 - scipy=1.14.1
 - seaborn=0.13.2
 - seaborn-base=0.13.2
@@ -497,6 +501,7 @@ dependencies:
 - statsmodels=0.14.4
 - stopit=1.1.2
 - tabulate=0.9.0
+- tbb=2022.0.0
 - tblib=3.0.0
 - terminado=0.18.1
 - texttable=1.7.0

envs/macos-pinned.yaml

@@ -89,6 +89,7 @@ dependencies:
 - contourpy=1.3.1
 - country_converter=1.2
 - cpp-expected=1.1.0
+- cppad=20240000.7
 - cycler=0.12.1
 - cyrus-sasl=2.1.27
 - cytoolz=1.0.1
@@ -243,6 +244,7 @@ dependencies:
 - libgoogle-cloud=2.32.0
 - libgoogle-cloud-storage=2.32.0
 - libgrpc=1.67.1
+- libhwloc=2.11.2
 - libiconv=1.17
 - libintl=0.22.5
 - libjpeg-turbo=3.0.0
@@ -303,6 +305,7 @@ dependencies:
 - metis=5.1.0
 - minizip=4.0.7
 - mistune=3.0.2
+- mpfr=4.2.1
 - msgpack-python=1.1.0
 - multipledispatch=0.6.0
 - multiurl=0.3.3
@@ -382,11 +385,12 @@ dependencies:
 - pyobjc-core=10.3.2
 - pyobjc-framework-cocoa=10.3.2
 - pyogrio=0.10.0
-- pyomo=6.8.2
+- pyomo=6.6.1
 - pyparsing=3.2.0
 - pyppmd=1.1.0
 - pyproj=3.7.0
-- pypsa=0.24.0
+- pypsa=0.28.0
+- pyscipopt=5.2.1
 - pyshp=2.3.1
 - pysocks=1.7.1
 - pytables=3.10.1
@@ -420,6 +424,7 @@ dependencies:
 - ruamel.yaml=0.17.26
 - ruamel.yaml.clib=0.2.8
 - scikit-learn=1.6.0
+- scip=9.2.0
 - scipy=1.14.1
 - seaborn=0.13.2
 - seaborn-base=0.13.2
@@ -442,6 +447,7 @@ dependencies:
 - statsmodels=0.14.4
 - stopit=1.1.2
 - tabulate=0.9.0
+- tbb=2022.0.0
 - tblib=3.0.0
 - terminado=0.18.1
 - texttable=1.7.0
@@ -480,7 +486,6 @@ dependencies:
 - xarray=2023.11.0
 - xerces-c=3.2.5
 - xlrd=2.0.1
-- xorg-libxau=1.0.12
 - xorg-libxdmcp=1.1.5
 - xyzservices=2024.9.0
 - yaml=0.2.5

envs/windows-pinned.yaml

@@ -82,6 +82,7 @@ dependencies:
 - contourpy=1.3.1
 - country_converter=1.2
 - cpp-expected=1.1.0
+- cppad=20240000.7
 - cpython=3.10.16
 - cycler=0.12.1
 - cytoolz=1.0.1
@@ -140,6 +141,7 @@ dependencies:
 - glib=2.82.2
 - glib-tools=2.82.2
 - glpk=5.0
+- gmp=6.3.0
 - graphite2=1.3.13
 - graphviz=12.0.0
 - gst-plugins-base=1.24.7
@@ -200,6 +202,7 @@ dependencies:
 - libarrow-dataset=18.1.0
 - libarrow-substrait=18.1.0
 - libblas=3.9.0
+- libboost=1.86.0
 - libbrotlicommon=1.1.0
 - libbrotlidec=1.1.0
 - libbrotlienc=1.1.0
@@ -233,6 +236,7 @@ dependencies:
 - libgoogle-cloud=2.32.0
 - libgoogle-cloud-storage=2.32.0
 - libgrpc=1.67.1
+- libhwloc=2.11.2
 - libiconv=1.17
 - libintl=0.22.5
 - libintl-devel=0.22.5
@@ -290,6 +294,7 @@ dependencies:
 - mercantile=1.2.1
 - minizip=4.0.7
 - mistune=3.0.2
+- mpfr=4.2.1
 - msgpack-python=1.1.0
 - multipledispatch=0.6.0
 - multiurl=0.3.3
@@ -361,14 +366,15 @@ dependencies:
 - pydoe2=1.3.0
 - pygments=2.18.0
 - pyogrio=0.10.0
-- pyomo=6.8.2
+- pyomo=6.6.1
 - pyparsing=3.2.0
 - pyppmd=1.1.0
 - pyproj=3.7.0
-- pypsa=0.24.0
+- pypsa=0.28.0
 - pyqt=5.15.9
 - pyqt5-sip=12.12.2
 - pyreadline3=3.5.4
+- pyscipopt=5.2.1
 - pyshp=2.3.1
 - pysocks=1.7.1
 - pytables=3.10.1
@@ -404,6 +410,7 @@ dependencies:
 - ruamel.yaml=0.17.26
 - ruamel.yaml.clib=0.2.8
 - scikit-learn=1.6.0
+- scip=9.2.0
 - scipy=1.14.1
 - seaborn=0.13.2
 - seaborn-base=0.13.2
@@ -427,6 +434,7 @@ dependencies:
 - statsmodels=0.14.4
 - stopit=1.1.2
 - tabulate=0.9.0
+- tbb=2022.0.0
 - tblib=3.0.0
 - terminado=0.18.1
 - texttable=1.7.0

scripts/_helpers.py

@@ -921,6 +921,16 @@ def get_last_commit_message(path):
     return last_commit_message
 
 
+def update_config_dictionary(
+    config_dict,
+    parameter_key_to_fill="lines",
+    dict_to_use={"geometry": "first", "bounds": "first"},
+):
+    config_dict.setdefault(parameter_key_to_fill, {})
+    config_dict[parameter_key_to_fill].update(dict_to_use)
+    return config_dict
+
+
 # PYPSA-EARTH-SEC
 def annuity(n, r):
     """

scripts/base_network.py

@@ -523,20 +523,6 @@ def base_network(
             result_type="reduce",
         )
         n.import_components_from_dataframe(lines_ac, "Line")
-        # The columns which names starts with "bus" are mixed up with the third-bus specification
-        # when executing additional_linkports()
-        lines_dc.drop(
-            labels=[
-                "bus0_lon",
-                "bus0_lat",
-                "bus1_lon",
-                "bus1_lat",
-                "bus_0_coors",
-                "bus_1_coors",
-            ],
-            axis=1,
-            inplace=True,
-        )
         n.import_components_from_dataframe(lines_dc, "Link")
 
     n.import_components_from_dataframe(transformers, "Transformer")

scripts/build_osm_network.py

@@ -26,6 +26,27 @@
 logger = create_logger(__name__)
 
 
+# Keep only a predefined set of columns, as otherwise conflicts are possible
+# e.g. the columns which names starts with "bus" are mixed up with
+# the third-bus specification when executing additional_linkports()
+LINES_COLUMNS = [
+    "line_id",
+    "circuits",
+    "tag_type",
+    "voltage",
+    "bus0",
+    "bus1",
+    "length",
+    "underground",
+    "under_construction",
+    "tag_frequency",
+    "dc",
+    "country",
+    "geometry",
+    "bounds",
+]
+
+
 def line_endings_to_bus_conversion(lines):
     # Assign to every line a start and end point
 
@@ -720,6 +741,7 @@ def built_network(
     countries_config,
     geo_crs,
     distance_crs,
+    lines_cols_standard,
     force_ac=False,
 ):
     logger.info("Stage 1/5: Read input data")
@@ -784,6 +806,8 @@ def built_network(
     if not os.path.exists(outputs["lines"]):
         os.makedirs(os.path.dirname(outputs["lines"]), exist_ok=True)
 
+    lines = lines[lines_cols_standard]
+
     to_csv_nafix(lines, outputs["lines"])  # Generate CSV
     to_csv_nafix(converters, outputs["converters"])  # Generate CSV
     to_csv_nafix(transformers, outputs["transformers"])  # Generate CSV
@@ -819,5 +843,6 @@ def built_network(
         countries,
         geo_crs,
         distance_crs,
+        lines_cols_standard=LINES_COLUMNS,
         force_ac=force_ac,
     )