From 673ad1fb585b50eaa28837dd5ac2ebb716996fa4 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 10:31:46 +0200
Subject: [PATCH 01/18] Snakefile: add missing folders that are retrieved

---
 Snakefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Snakefile b/Snakefile
index d428db45..d3be3f4f 100644
--- a/Snakefile
+++ b/Snakefile
@@ -52,6 +52,10 @@ datafiles = [
     "Industrial_Database.csv",
     "emobility/KFZ__count",
     "emobility/Pkw__count",
+    "eurostat-energy_balances-june_2016_edition",
+    "eurostat-energy_balances-may_2018_edition"
+    "jrc-idees-2015",
+    "h2_salt_caverns_GWh_per_sqkm.geojson",
 ]
 
 if config.get('retrieve_sector_databundle', True):

From 7b4aab7c3f3bad442c512c65b82087de4cac70b4 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 10:32:24 +0200
Subject: [PATCH 02/18] Snakefile: fix syntax error

---
 Snakefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Snakefile b/Snakefile
index d3be3f4f..329e03e9 100644
--- a/Snakefile
+++ b/Snakefile
@@ -53,7 +53,7 @@ datafiles = [
     "emobility/KFZ__count",
     "emobility/Pkw__count",
     "eurostat-energy_balances-june_2016_edition",
-    "eurostat-energy_balances-may_2018_edition"
+    "eurostat-energy_balances-may_2018_edition",
     "jrc-idees-2015",
     "h2_salt_caverns_GWh_per_sqkm.geojson",
 ]

From 66cddee62d7a6ada0f3bd00b63b72ef4572cff9e Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:26:31 +0200
Subject: [PATCH 03/18] Add basic CI

---
 .github/workflows/ci.yaml  |  95 ++++++
 test/config.myopic.yaml    | 607 +++++++++++++++++++++++++++++++++++++
 test/config.overnight.yaml | 605 ++++++++++++++++++++++++++++++++++++
 3 files changed, 1307 insertions(+)
 create mode 100644 .github/workflows/ci.yaml
 create mode 100644 test/config.myopic.yaml
 create mode 100644 test/config.overnight.yaml

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 00000000..7e032126
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: : 2021 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+name: CI
+
+# Caching method based on and described by:
+# epassaro (2021): https://dev.to/epassaro/caching-anaconda-environments-in-github-actions-5hde
+# and code in GitHub repo: https://github.com/epassaro/cache-conda-envs
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+  schedule:
+    - cron: "0 5 * * TUE"
+
+env:
+  CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        include:
+          # Matrix required to handle caching with Mambaforge
+          - os: ubuntu-latest
+            label: ubuntu-latest
+            prefix: /usr/share/miniconda3/envs/pypsa-eur
+
+          # - os: macos-latest
+          #   label: macos-latest
+          #   prefix: /Users/runner/miniconda3/envs/pypsa-eur
+
+          # - os: windows-latest
+          #   label: windows-latest
+          #   prefix: C:\Miniconda3\envs\pypsa-eur
+
+    name: ${{ matrix.label }}
+
+    runs-on: ${{ matrix.os }}
+    
+    defaults:
+      run:
+        shell: bash -l {0}
+        
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Clone pypsa-eur and technology-data repositories
+        run: |
+          git clone https://github.com/pypsa/pypsa-eur ../pypsa-eur
+          git clone https://github.com/pypsa/technology-data ../technology-data
+          cp ../pypsa-eur/test/config.test1.yaml  ../pypsa-eur/config.yaml
+      
+      - name: Setup secrets
+        run: |
+          echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc
+
+      - name: Add solver to environment
+        run: |
+          echo -e "  - glpk\n  - ipopt<3.13.3" >> ../pypsa-eur/envs/environment.yaml
+
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: pypsa-eur
+            use-mamba: true
+            
+      - name: Set cache date
+        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+
+      - name: Create environment cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: ${{ matrix.prefix }}
+          key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+
+      - name: Update environment due to outdated or unavailable cache
+        run: mamba env update -n pypsa-eur -f envs/environment.yaml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Test snakemake workflow
+        run: |
+          conda activate pypsa-eur
+          conda list
+          snakemake -jall --configfile test/config.overnight.yaml solve_all_networks
+          snakemake -jall --configfile test/config.myopic.yaml solve_all_networks
diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml
new file mode 100644
index 00000000..ab8b0765
--- /dev/null
+++ b/test/config.myopic.yaml
@@ -0,0 +1,607 @@
+version: 0.6.0
+
+logging_level: INFO
+
+retrieve_sector_databundle: true
+
+results_dir: results/
+summary_dir: results
+costs_dir: ../technology-data/outputs/
+run: test-myopic  # use this to keep track of runs with different settings
+foresight: myopic # options are overnight, myopic, perfect (perfect is not yet implemented)
+# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
+
+scenario:
+  simpl: # only relevant for PyPSA-Eur
+    - ''
+  lv: # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt"
+    - 1.5
+  clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
+    - 5
+  opts: # only relevant for PyPSA-Eur
+    - ''
+  sector_opts: # this is where the main scenario settings are
+    - 191H-T-H-B-I-A-solar+p3-dist1
+  # to really understand the options here, look in scripts/prepare_sector_network.py
+  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
+  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
+  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
+  # single letters are sectors: T for land transport, H for building heating,
+  # B for biomass supply, I for industry, shipping and aviation,
+  # A for agriculture, forestry and fishing
+  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
+  # solar+p3 multiplies the available installable potential by factor 3
+  # co2 stored+e2 multiplies the potential of CO2 sequestration by a factor 2
+  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
+  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
+  # emissions throughout the transition path in the timeframe determined by the
+  # planning_horizons), be:beta decay; ex:exponential decay
+  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
+  # decay with initial growth rate 0
+  planning_horizons: # investment years for myopic and perfect; or costs year for overnight
+    - 2030
+    - 2040
+    - 2050
+  # for example, set to [2020, 2030, 2040, 2050] for myopic foresight
+
+# CO2 budget as a fraction of 1990 emissions
+# this is over-ridden if CO2Lx is set in sector_opts
+# this is also over-ridden if cb is set in sector_opts
+co2_budget:
+  2020: 0.7011648746
+  2025: 0.5241935484
+  2030: 0.2970430108
+  2035: 0.1500896057
+  2040: 0.0712365591
+  2045: 0.0322580645
+  2050: 0
+
+# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec
+snapshots:
+  # arguments to pd.date_range
+  start: "2013-03-01"
+  end: "2013-04-01"
+  closed: left # end is not inclusive
+
+atlite:
+  cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc
+
+# this information is NOT used but needed as an argument for
+# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py
+electricity:
+  max_hours:
+    battery: 6
+    H2: 168
+
+# regulate what components with which carriers are kept from PyPSA-Eur;
+# some technologies are removed because they are implemented differently
+# (e.g. battery or H2 storage) or have different year-dependent costs
+# in PyPSA-Eur-Sec
+pypsa_eur:
+  Bus:
+    - AC
+  Link:
+    - DC
+  Generator:
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - solar
+    - ror
+  StorageUnit:
+    - PHS
+    - hydro
+  Store: []
+
+
+energy:
+  energy_totals_year: 2011
+  base_emissions_year: 1990
+  eurostat_report_year: 2016
+  emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
+
+biomass:
+  year: 2030
+  scenario: ENS_Med
+  classes:
+    solid biomass:
+      - Agricultural waste
+      - Fuelwood residues
+      - Secondary Forestry residues - woodchips
+      - Sawdust
+      - Residues from landscape care
+      - Municipal waste
+    not included:
+      - Sugar from sugar beet
+      - Rape seed
+      - "Sunflower, soya seed "
+      - Bioethanol barley, wheat, grain maize, oats, other cereals and rye
+      - Miscanthus, switchgrass, RCG
+      - Willow
+      - Poplar
+      - FuelwoodRW
+      - C&P_RW
+    biogas:
+      - Manure solid, liquid
+      - Sludge
+
+
+solar_thermal:
+  clearsky_model: simple  # should be "simple" or "enhanced"?
+  orientation:
+    slope: 45.
+    azimuth: 180.
+
+# only relevant for foresight = myopic or perfect
+existing_capacities:
+  grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019]
+  threshold_capacity: 10
+  conventional_carriers:
+    - lignite
+    - coal
+    - oil
+    - uranium
+
+
+sector:
+  district_heating:
+    potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
+     # increase of today's district heating demand to potential maximum district heating share
+     # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
+    progress: 1
+      # 2020: 0.0
+      # 2030: 0.3
+      # 2040: 0.6
+      # 2050: 1.0
+    district_heating_loss: 0.15
+  bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
+  bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
+  transport_heating_deadband_upper: 20.
+  transport_heating_deadband_lower: 15.
+  ICE_lower_degree_factor: 0.375  #in per cent increase in fuel consumption per degree above deadband
+  ICE_upper_degree_factor: 1.6
+  EV_lower_degree_factor: 0.98
+  EV_upper_degree_factor: 0.63
+  bev_dsm: true #turns on EV battery
+  bev_availability: 0.5  #How many cars do smart charging
+  bev_energy: 0.05  #average battery size in MWh
+  bev_charge_efficiency: 0.9  #BEV (dis-)charging efficiency
+  bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S
+  bev_charge_rate: 0.011 #3-phase charger with 11 kW
+  bev_avail_max: 0.95
+  bev_avail_mean: 0.8
+  v2g: true #allows feed-in to grid from EV battery
+  #what is not EV or FCEV is oil-fuelled ICE
+  land_transport_fuel_cell_share: 0.15 # 1 means all FCEVs
+    # 2020: 0
+    # 2030: 0.05
+    # 2040: 0.1
+    # 2050: 0.15
+  land_transport_electric_share: 0.85 # 1 means all EVs
+    # 2020: 0
+    # 2030: 0.25
+    # 2040: 0.6
+    # 2050: 0.85
+  transport_fuel_cell_efficiency: 0.5
+  transport_internal_combustion_efficiency: 0.3
+  agriculture_machinery_electric_share: 0
+  agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use
+  agriculture_machinery_electric_efficiency: 0.3 # electricity per use
+  shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011
+  shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
+  shipping_hydrogen_share: 1 # 1 means all hydrogen FC
+    # 2020: 0
+    # 2025: 0
+    # 2030: 0.05
+    # 2035: 0.15
+    # 2040: 0.3
+    # 2045: 0.6
+    # 2050: 1
+  time_dep_hp_cop: true #time dependent heat pump coefficient of performance
+  heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
+   # conservatively high to cover hot water and space heating in poorly-insulated buildings
+  reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
+  # this can represent e.g. building renovation, building demolition, or if
+  # the factor is negative: increasing floor area, increased thermal comfort, population growth
+  reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand
+  # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
+    # 2020: 0.10  # this results in a space heat demand reduction of 10%
+    # 2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
+    # 2030: 0.09
+    # 2035: 0.11
+    # 2040: 0.16
+    # 2045: 0.21
+    # 2050: 0.29
+  retrofitting :  # co-optimises building renovation to reduce space heat demand
+    retro_endogen: false  # co-optimise space heat savings
+    cost_factor: 1.0   # weight costs for building renovation
+    interest_rate: 0.04  # for investment in building components
+    annualise_cost: true  # annualise the investment costs
+    tax_weighting: false   # weight costs depending on taxes in countries
+    construction_index: true   # weight costs depending on labour/material costs per country
+  tes: true
+  tes_tau: # 180 day time constant for centralised, 3 day for decentralised
+    decentral: 3
+    central: 180
+  boilers: true
+  oil_boilers: false
+  chp: true
+  micro_chp: false
+  solar_thermal: true
+  solar_cf_correction: 0.788457  # =  >>> 1/1.2683
+  marginal_cost_storage: 0. #1e-4
+  methanation: true
+  helmeth: true
+  dac: true
+  co2_vent: true
+  SMR: true
+  co2_sequestration_potential: 200  #MtCO2/a sequestration potential for Europe
+  co2_sequestration_cost: 10   #EUR/tCO2 for sequestration of CO2
+  co2_network: false
+  cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
+  hydrogen_underground_storage: true
+  hydrogen_underground_storage_locations:
+    # - onshore  # more than 50 km from sea
+    - nearshore  # within 50 km of sea
+    # - offshore
+  use_fischer_tropsch_waste_heat: true
+  use_fuel_cell_waste_heat: true
+  electricity_distribution_grid: true
+  electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  electricity_grid_connection: true  # only applies to onshore wind and utility PV
+  H2_network: true
+  gas_network: false
+  H2_retrofit: false  # if set to True existing gas pipes can be retrofitted to H2 pipes
+  # according to hydrogen backbone strategy (April, 2020) p.15
+  # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
+  # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity
+  H2_retrofit_capacity_per_CH4: 0.6  # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
+  gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation
+  gas_distribution_grid: true
+  gas_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  biomass_transport: false  # biomass transport between nodes
+  conventional_generation: # generator : carrier
+    OCGT: gas
+
+
+industry:
+  St_primary_fraction: 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
+    # 2020: 0.6
+    # 2025: 0.55
+    # 2030: 0.5
+    # 2035: 0.45
+    # 2040: 0.4
+    # 2045: 0.35
+    # 2050: 0.3
+  DRI_fraction: 1 # fraction of the primary route converted to DRI + EAF
+    # 2020: 0
+    # 2025: 0
+    # 2030: 0.05
+    # 2035: 0.2
+    # 2040: 0.4
+    # 2045: 0.7
+    # 2050: 1
+  H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
+  elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
+  Al_primary_fraction: 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+    # 2020: 0.4
+    # 2025: 0.375
+    # 2030: 0.35
+    # 2035: 0.325
+    # 2040: 0.3
+    # 2045: 0.25
+    # 2050: 0.2
+  MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+  MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
+  MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
+  MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
+  NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
+  petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
+  HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route
+  HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling
+  HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling
+  HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX
+  MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756.
+  MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking
+  chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43
+  MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43
+  MWh_H2_per_tCl: -0.9372  # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process
+  methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62
+  MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65
+  MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65
+  hotmaps_locate_missing: false
+  reference_year: 2015
+  # references:
+  # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
+  # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
+
+costs:
+  lifetime: 25 #default lifetime
+  # From a Lion Hirth paper, also reflects average of Noothout et al 2016
+  discountrate: 0.07
+  # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501
+  USD2013_to_EUR2013: 0.7532
+
+  # Marginal and capital costs can be overwritten
+  # capital_cost:
+  #   onwind: 500
+  marginal_cost:
+    solar: 0.01
+    onwind: 0.015
+    offwind: 0.015
+    hydro: 0.
+    H2: 0.
+    battery: 0.
+
+  emission_prices: # only used with the option Ep (emission prices)
+    co2: 0.
+
+  lines:
+    length_factor: 1.25 #to estimate offwind connection costs
+
+
+solving:
+  #tmpdir: "path/to/tmp"
+  options:
+    formulation: kirchhoff
+    clip_p_max_pu: 1.e-2
+    load_shedding: false
+    noisy_costs: true
+    skip_iterations: true
+    track_iterations: false
+    min_iterations: 4
+    max_iterations: 6
+    keep_shadowprices:
+      - Bus
+      - Line
+      - Link
+      - Transformer
+      - GlobalConstraint
+      - Generator
+      - Store
+      - StorageUnit
+
+  solver:
+    name: cbc
+    # threads: 4
+    # method: 2 # barrier
+    # crossover: 0
+    # BarConvTol: 1.e-6
+    # Seed: 123
+    # AggFill: 0
+    # PreDual: 0
+    # GURO_PAR_BARDENSETHRESH: 200
+    #FeasibilityTol: 1.e-6
+
+    #name: cplex
+    #threads: 4
+    #lpmethod: 4 # barrier
+    #solutiontype: 2 # non basic solution, ie no crossover
+    #barrier_convergetol: 1.e-5
+    #feasopt_tolerance: 1.e-6
+  mem: 4000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+
+
+plotting:
+  map:
+    boundaries: [-11, 30, 34, 71]
+    color_geomap:
+      ocean: white
+      land: whitesmoke
+  costs_max: 1000
+  costs_threshold: 1
+  energy_max: 20000
+  energy_min: -20000
+  energy_threshold: 50
+  vre_techs:
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - solar
+    - ror
+  renewable_storage_techs:
+    - PHS
+    - hydro
+  conv_techs:
+    - OCGT
+    - CCGT
+    - Nuclear
+    - Coal
+  storage_techs:
+    - hydro+PHS
+    - battery
+    - H2
+  load_carriers:
+    - AC load
+  AC_carriers:
+    - AC line
+    - AC transformer
+  link_carriers:
+    - DC line
+    - Converter AC-DC
+  heat_links:
+    - heat pump
+    - resistive heater
+    - CHP heat
+    - CHP electric
+    - gas boiler
+    - central heat pump
+    - central resistive heater
+    - central CHP heat
+    - central CHP electric
+    - central gas boiler
+  heat_generators:
+    - gas boiler
+    - central gas boiler
+    - solar thermal collector
+    - central solar thermal collector
+  tech_colors:
+    # wind
+    onwind: "#235ebc"
+    onshore wind: "#235ebc"
+    offwind: "#6895dd"
+    offshore wind: "#6895dd"
+    offwind-ac: "#6895dd"
+    offshore wind (AC): "#6895dd"
+    offwind-dc: "#74c6f2"
+    offshore wind (DC): "#74c6f2"
+    # water
+    hydro: '#298c81'
+    hydro reservoir: '#298c81'
+    ror: '#3dbfb0'
+    run of river: '#3dbfb0'
+    hydroelectricity: '#298c81'
+    PHS: '#51dbcc'
+    wave: '#a7d4cf'
+    # solar
+    solar: "#f9d002"
+    solar PV: "#f9d002"
+    solar thermal: '#ffbf2b'
+    solar rooftop: '#ffea80'
+    # gas
+    OCGT: '#e0986c'
+    OCGT marginal: '#e0986c'
+    OCGT-heat: '#e0986c'
+    gas boiler: '#db6a25'
+    gas boilers: '#db6a25'
+    gas boiler marginal: '#db6a25'
+    gas: '#e05b09'
+    fossil gas: '#e05b09'
+    natural gas: '#e05b09'
+    CCGT: '#a85522'
+    CCGT marginal: '#a85522'
+    gas for industry co2 to atmosphere: '#692e0a'
+    gas for industry co2 to stored: '#8a3400'
+    gas for industry: '#853403'
+    gas for industry CC: '#692e0a'
+    gas pipeline: '#ebbca0'
+    gas pipeline new: '#a87c62'
+    # oil
+    oil: '#c9c9c9'
+    oil boiler: '#adadad'
+    agriculture machinery oil: '#949494'
+    shipping oil: "#808080"
+    land transport oil: '#afafaf'
+    # nuclear
+    Nuclear: '#ff8c00'
+    Nuclear marginal: '#ff8c00'
+    nuclear: '#ff8c00'
+    uranium: '#ff8c00'
+    # coal
+    Coal: '#545454'
+    coal: '#545454'
+    Coal marginal: '#545454'
+    solid: '#545454'
+    Lignite: '#826837'
+    lignite: '#826837'
+    Lignite marginal: '#826837'
+    # biomass
+    biogas: '#e3d37d'
+    biomass: '#baa741'
+    solid biomass: '#baa741'
+    solid biomass transport: '#baa741'
+    solid biomass for industry: '#7a6d26'
+    solid biomass for industry CC: '#47411c'
+    solid biomass for industry co2 from atmosphere: '#736412'
+    solid biomass for industry co2 to stored: '#47411c'
+    # power transmission
+    lines: '#6c9459'
+    transmission lines: '#6c9459'
+    electricity distribution grid: '#97ad8c'
+    # electricity demand
+    Electric load: '#110d63'
+    electric demand: '#110d63'
+    electricity: '#110d63'
+    industry electricity: '#2d2a66'
+    industry new electricity: '#2d2a66'
+    agriculture electricity: '#494778'
+    # battery + EVs
+    battery: '#ace37f'
+    battery storage: '#ace37f'
+    home battery: '#80c944'
+    home battery storage: '#80c944'
+    BEV charger: '#baf238'
+    V2G: '#e5ffa8'
+    land transport EV: '#baf238'
+    Li ion: '#baf238'
+    # hot water storage
+    water tanks: '#e69487'
+    hot water storage: '#e69487'
+    hot water charging: '#e69487'
+    hot water discharging: '#e69487'
+    # heat demand
+    Heat load: '#cc1f1f'
+    heat: '#cc1f1f'
+    heat demand: '#cc1f1f'
+    rural heat: '#ff5c5c'
+    central heat: '#cc1f1f'
+    decentral heat: '#750606'
+    low-temperature heat for industry: '#8f2727'
+    process heat: '#ff0000'
+    agriculture heat: '#d9a5a5'
+    # heat supply
+    heat pumps: '#2fb537'
+    heat pump: '#2fb537'
+    air heat pump: '#36eb41'
+    ground heat pump: '#2fb537'
+    Ambient: '#98eb9d'
+    CHP: '#8a5751'
+    CHP CC: '#634643'
+    CHP heat: '#8a5751'
+    CHP electric: '#8a5751'
+    district heating: '#e8beac'
+    resistive heater: '#d8f9b8'
+    retrofitting: '#8487e8'
+    building retrofitting: '#8487e8'
+    # hydrogen
+    H2 for industry: "#f073da"
+    H2 for shipping: "#ebaee0"
+    H2: '#bf13a0'
+    hydrogen: '#bf13a0'
+    SMR: '#870c71'
+    SMR CC: '#4f1745'
+    H2 liquefaction: '#d647bd'
+    hydrogen storage: '#bf13a0'
+    H2 storage: '#bf13a0'
+    land transport fuel cell: '#6b3161'
+    H2 pipeline: '#f081dc'
+    H2 pipeline retrofitted: '#ba99b5'
+    H2 Fuel Cell: '#c251ae'
+    H2 Electrolysis: '#ff29d9'
+    # syngas
+    Sabatier: '#9850ad'
+    methanation: '#c44ce6'
+    methane: '#c44ce6'
+    helmeth: '#e899ff'
+    # synfuels
+    Fischer-Tropsch: '#25c49a'
+    liquid: '#25c49a'
+    kerosene for aviation: '#a1ffe6'
+    naphtha for industry: '#57ebc4'
+    # co2
+    CC: '#f29dae'
+    CCS: '#f29dae'
+    CO2 sequestration: '#f29dae'
+    DAC: '#ff5270'
+    co2 stored: '#f2385a'
+    co2: '#f29dae'
+    co2 vent: '#ffd4dc'
+    CO2 pipeline: '#f5627f'
+    # emissions
+    process emissions CC: '#000000'
+    process emissions: '#222222'
+    process emissions to stored: '#444444'
+    process emissions to atmosphere: '#888888'
+    oil emissions: '#aaaaaa'
+    shipping oil emissions: "#555555"
+    land transport oil emissions: '#777777'
+    agriculture machinery oil emissions: '#333333'
+    # other
+    shipping: '#03a2ff'
+    power-to-heat: '#2fb537'
+    power-to-gas: '#c44ce6'
+    power-to-H2: '#ff29d9'
+    power-to-liquid: '#25c49a'
+    gas-to-power/heat: '#ee8340'
+    waste: '#e3d37d'
+    other: '#000000'
diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml
new file mode 100644
index 00000000..3764c8fa
--- /dev/null
+++ b/test/config.overnight.yaml
@@ -0,0 +1,605 @@
+version: 0.6.0
+
+logging_level: INFO
+
+retrieve_sector_databundle: true
+
+results_dir: results/
+summary_dir: results
+costs_dir: ../technology-data/outputs/
+run: test-overnight  # use this to keep track of runs with different settings
+foresight: overnight # options are overnight, myopic, perfect (perfect is not yet implemented)
+# if you use myopic or perfect foresight, set the investment years in "planning_horizons" below
+
+scenario:
+  simpl: # only relevant for PyPSA-Eur
+    - ''
+  lv: # allowed transmission line volume expansion, can be any float >= 1.0 (today) or "opt"
+    - 1.5
+  clusters: # number of nodes in Europe, any integer between 37 (1 node per country-zone) and several hundred
+    - 5
+  opts: # only relevant for PyPSA-Eur
+    - ''
+  sector_opts: # this is where the main scenario settings are
+    - CO2L0-191H-T-H-B-I-A-solar+p3-dist1
+  # to really understand the options here, look in scripts/prepare_sector_network.py
+  # Co2Lx specifies the CO2 target in x% of the 1990 values; default will give default (5%);
+  # Co2L0p25 will give 25% CO2 emissions; Co2Lm0p05 will give 5% negative emissions
+  # xH is the temporal resolution; 3H is 3-hourly, i.e. one snapshot every 3 hours
+  # single letters are sectors: T for land transport, H for building heating,
+  # B for biomass supply, I for industry, shipping and aviation,
+  # A for agriculture, forestry and fishing
+  # solar+c0.5 reduces the capital cost of solar to 50\% of reference value
+  # solar+p3 multiplies the available installable potential by factor 3
+  # co2 stored+e2 multiplies the potential of CO2 sequestration by a factor 2
+  # dist{n} includes distribution grids with investment cost of n times cost in data/costs.csv
+  # for myopic/perfect foresight cb states the carbon budget in GtCO2 (cumulative
+  # emissions throughout the transition path in the timeframe determined by the
+  # planning_horizons), be:beta decay; ex:exponential decay
+  # cb40ex0 distributes a carbon budget of 40 GtCO2 following an exponential
+  # decay with initial growth rate 0
+  planning_horizons: # investment years for myopic and perfect; or costs year for overnight
+    - 2030
+  # for example, set to [2020, 2030, 2040, 2050] for myopic foresight
+
+# CO2 budget as a fraction of 1990 emissions
+# this is over-ridden if CO2Lx is set in sector_opts
+# this is also over-ridden if cb is set in sector_opts
+co2_budget:
+  2020: 0.7011648746
+  2025: 0.5241935484
+  2030: 0.2970430108
+  2035: 0.1500896057
+  2040: 0.0712365591
+  2045: 0.0322580645
+  2050: 0
+
+# snapshots are originally set in PyPSA-Eur/config.yaml but used again by PyPSA-Eur-Sec
+snapshots:
+  # arguments to pd.date_range
+  start: "2013-03-01"
+  end: "2013-04-01"
+  closed: left # end is not inclusive
+
+atlite:
+  cutout: ../pypsa-eur/cutouts/be-03-2013-era5.nc
+
+# this information is NOT used but needed as an argument for
+# pypsa-eur/scripts/add_electricity.py/load_costs in make_summary.py
+electricity:
+  max_hours:
+    battery: 6
+    H2: 168
+
+# regulate what components with which carriers are kept from PyPSA-Eur;
+# some technologies are removed because they are implemented differently
+# (e.g. battery or H2 storage) or have different year-dependent costs
+# in PyPSA-Eur-Sec
+pypsa_eur:
+  Bus:
+    - AC
+  Link:
+    - DC
+  Generator:
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - solar
+    - ror
+  StorageUnit:
+    - PHS
+    - hydro
+  Store: []
+
+
+energy:
+  energy_totals_year: 2011
+  base_emissions_year: 1990
+  eurostat_report_year: 2016
+  emissions: CO2 # "CO2" or "All greenhouse gases - (CO2 equivalent)"
+
+biomass:
+  year: 2030
+  scenario: ENS_Med
+  classes:
+    solid biomass:
+      - Agricultural waste
+      - Fuelwood residues
+      - Secondary Forestry residues - woodchips
+      - Sawdust
+      - Residues from landscape care
+      - Municipal waste
+    not included:
+      - Sugar from sugar beet
+      - Rape seed
+      - "Sunflower, soya seed "
+      - Bioethanol barley, wheat, grain maize, oats, other cereals and rye
+      - Miscanthus, switchgrass, RCG
+      - Willow
+      - Poplar
+      - FuelwoodRW
+      - C&P_RW
+    biogas:
+      - Manure solid, liquid
+      - Sludge
+
+
+solar_thermal:
+  clearsky_model: simple  # should be "simple" or "enhanced"?
+  orientation:
+    slope: 45.
+    azimuth: 180.
+
+# only relevant for foresight = myopic or perfect
+existing_capacities:
+  grouping_years: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019]
+  threshold_capacity: 10
+  conventional_carriers:
+    - lignite
+    - coal
+    - oil
+    - uranium
+
+
+sector:
+  district_heating:
+    potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
+     # increase of today's district heating demand to potential maximum district heating share
+     # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
+    progress: 1
+      # 2020: 0.0
+      # 2030: 0.3
+      # 2040: 0.6
+      # 2050: 1.0
+    district_heating_loss: 0.15
+  bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
+  bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
+  transport_heating_deadband_upper: 20.
+  transport_heating_deadband_lower: 15.
+  ICE_lower_degree_factor: 0.375  #in per cent increase in fuel consumption per degree above deadband
+  ICE_upper_degree_factor: 1.6
+  EV_lower_degree_factor: 0.98
+  EV_upper_degree_factor: 0.63
+  bev_dsm: true #turns on EV battery
+  bev_availability: 0.5  #How many cars do smart charging
+  bev_energy: 0.05  #average battery size in MWh
+  bev_charge_efficiency: 0.9  #BEV (dis-)charging efficiency
+  bev_plug_to_wheel_efficiency: 0.2 #kWh/km from EPA https://www.fueleconomy.gov/feg/ for Tesla Model S
+  bev_charge_rate: 0.011 #3-phase charger with 11 kW
+  bev_avail_max: 0.95
+  bev_avail_mean: 0.8
+  v2g: true #allows feed-in to grid from EV battery
+  #what is not EV or FCEV is oil-fuelled ICE
+  land_transport_fuel_cell_share: 0.15 # 1 means all FCEVs
+    # 2020: 0
+    # 2030: 0.05
+    # 2040: 0.1
+    # 2050: 0.15
+  land_transport_electric_share: 0.85 # 1 means all EVs
+    # 2020: 0
+    # 2030: 0.25
+    # 2040: 0.6
+    # 2050: 0.85
+  transport_fuel_cell_efficiency: 0.5
+  transport_internal_combustion_efficiency: 0.3
+  agriculture_machinery_electric_share: 0
+  agriculture_machinery_fuel_efficiency: 0.7 # fuel oil per use
+  agriculture_machinery_electric_efficiency: 0.3 # electricity per use
+  shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011
+  shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
+  shipping_hydrogen_share: 1 # 1 means all hydrogen FC
+    # 2020: 0
+    # 2025: 0
+    # 2030: 0.05
+    # 2035: 0.15
+    # 2040: 0.3
+    # 2045: 0.6
+    # 2050: 1
+  time_dep_hp_cop: true #time dependent heat pump coefficient of performance
+  heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
+   # conservatively high to cover hot water and space heating in poorly-insulated buildings
+  reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
+  # this can represent e.g. building renovation, building demolition, or if
+  # the factor is negative: increasing floor area, increased thermal comfort, population growth
+  reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand
+  # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
+    # 2020: 0.10  # this results in a space heat demand reduction of 10%
+    # 2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
+    # 2030: 0.09
+    # 2035: 0.11
+    # 2040: 0.16
+    # 2045: 0.21
+    # 2050: 0.29
+  retrofitting :  # co-optimises building renovation to reduce space heat demand
+    retro_endogen: false  # co-optimise space heat savings
+    cost_factor: 1.0   # weight costs for building renovation
+    interest_rate: 0.04  # for investment in building components
+    annualise_cost: true  # annualise the investment costs
+    tax_weighting: false   # weight costs depending on taxes in countries
+    construction_index: true   # weight costs depending on labour/material costs per country
+  tes: true
+  tes_tau: # 180 day time constant for centralised, 3 day for decentralised
+    decentral: 3
+    central: 180
+  boilers: true
+  oil_boilers: false
+  chp: true
+  micro_chp: false
+  solar_thermal: true
+  solar_cf_correction: 0.788457  # =  >>> 1/1.2683
+  marginal_cost_storage: 0. #1e-4
+  methanation: true
+  helmeth: true
+  dac: true
+  co2_vent: true
+  SMR: true
+  co2_sequestration_potential: 200  #MtCO2/a sequestration potential for Europe
+  co2_sequestration_cost: 10   #EUR/tCO2 for sequestration of CO2
+  co2_network: false
+  cc_fraction: 0.9  # default fraction of CO2 captured with post-combustion capture
+  hydrogen_underground_storage: true
+  hydrogen_underground_storage_locations:
+    # - onshore  # more than 50 km from sea
+    - nearshore  # within 50 km of sea
+    # - offshore
+  use_fischer_tropsch_waste_heat: true
+  use_fuel_cell_waste_heat: true
+  electricity_distribution_grid: true
+  electricity_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  electricity_grid_connection: true  # only applies to onshore wind and utility PV
+  H2_network: true
+  gas_network: true
+  H2_retrofit: true  # if set to True existing gas pipes can be retrofitted to H2 pipes
+  # according to hydrogen backbone strategy (April, 2020) p.15
+  # https://gasforclimate2050.eu/wp-content/uploads/2020/07/2020_European-Hydrogen-Backbone_Report.pdf
+  # 60% of original natural gas capacity could be used in cost-optimal case as H2 capacity
+  H2_retrofit_capacity_per_CH4: 0.6  # ratio for H2 capacity per original CH4 capacity of retrofitted pipelines
+  gas_network_connectivity_upgrade: 1 # https://networkx.org/documentation/stable/reference/algorithms/generated/networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation.html#networkx.algorithms.connectivity.edge_augmentation.k_edge_augmentation
+  gas_distribution_grid: true
+  gas_distribution_grid_cost_factor: 1.0  #multiplies cost in data/costs.csv
+  biomass_transport: false  # biomass transport between nodes
+  conventional_generation: # generator : carrier
+    OCGT: gas
+
+
+industry:
+  St_primary_fraction: 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
+    # 2020: 0.6
+    # 2025: 0.55
+    # 2030: 0.5
+    # 2035: 0.45
+    # 2040: 0.4
+    # 2045: 0.35
+    # 2050: 0.3
+  DRI_fraction: 1 # fraction of the primary route converted to DRI + EAF
+    # 2020: 0
+    # 2025: 0
+    # 2030: 0.05
+    # 2035: 0.2
+    # 2040: 0.4
+    # 2045: 0.7
+    # 2050: 1
+  H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
+  elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
+  Al_primary_fraction: 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+    # 2020: 0.4
+    # 2025: 0.375
+    # 2030: 0.35
+    # 2035: 0.325
+    # 2040: 0.3
+    # 2045: 0.25
+    # 2050: 0.2
+  MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
+  MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
+  MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)
+  MWh_elec_per_tNH3_electrolysis: 1.17 # from https://doi.org/10.1016/j.joule.2018.04.017 Table 13 (air separation and HB)
+  NH3_process_emissions: 24.5 # in MtCO2/a from SMR for H2 production for NH3 from UNFCCC for 2015 for EU28
+  petrochemical_process_emissions: 25.5 # in MtCO2/a for petrochemical and other from UNFCCC for 2015 for EU28
+  HVC_primary_fraction: 1. # fraction of today's HVC produced via primary route
+  HVC_mechanical_recycling_fraction: 0. # fraction of today's HVC produced via mechanical recycling
+  HVC_chemical_recycling_fraction: 0. # fraction of today's HVC produced via chemical recycling
+  HVC_production_today: 52. # MtHVC/a from DECHEMA (2017), Figure 16, page 107; includes ethylene, propylene and BTX
+  MWh_elec_per_tHVC_mechanical_recycling: 0.547 # from SI of https://doi.org/10.1016/j.resconrec.2020.105010, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756.
+  MWh_elec_per_tHVC_chemical_recycling: 6.9 # Material Economics (2019), page 125; based on pyrolysis and electric steam cracking
+  chlorine_production_today: 9.58 # MtCl/a from DECHEMA (2017), Table 7, page 43
+  MWh_elec_per_tCl: 3.6 # DECHEMA (2017), Table 6, page 43
+  MWh_H2_per_tCl: -0.9372  # DECHEMA (2017), page 43; negative since hydrogen produced in chloralkali process
+  methanol_production_today: 1.5 # MtMeOH/a from DECHEMA (2017), page 62
+  MWh_elec_per_tMeOH: 0.167 # DECHEMA (2017), Table 14, page 65
+  MWh_CH4_per_tMeOH: 10.25 # DECHEMA (2017), Table 14, page 65
+  hotmaps_locate_missing: false
+  reference_year: 2015
+  # references:
+  # DECHEMA (2017): https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf
+  # Material Economics (2019): https://materialeconomics.com/latest-updates/industrial-transformation-2050
+
+costs:
+  lifetime: 25 #default lifetime
+  # From a Lion Hirth paper, also reflects average of Noothout et al 2016
+  discountrate: 0.07
+  # [EUR/USD] ECB: https://www.ecb.europa.eu/stats/exchange/eurofxref/html/eurofxref-graph-usd.en.html # noqa: E501
+  USD2013_to_EUR2013: 0.7532
+
+  # Marginal and capital costs can be overwritten
+  # capital_cost:
+  #   onwind: 500
+  marginal_cost:
+    solar: 0.01
+    onwind: 0.015
+    offwind: 0.015
+    hydro: 0.
+    H2: 0.
+    battery: 0.
+
+  emission_prices: # only used with the option Ep (emission prices)
+    co2: 0.
+
+  lines:
+    length_factor: 1.25 #to estimate offwind connection costs
+
+
+solving:
+  #tmpdir: "path/to/tmp"
+  options:
+    formulation: kirchhoff
+    clip_p_max_pu: 1.e-2
+    load_shedding: false
+    noisy_costs: true
+    skip_iterations: true
+    track_iterations: false
+    min_iterations: 4
+    max_iterations: 6
+    keep_shadowprices:
+      - Bus
+      - Line
+      - Link
+      - Transformer
+      - GlobalConstraint
+      - Generator
+      - Store
+      - StorageUnit
+
+  solver:
+    name: cbc
+    # threads: 4
+    # method: 2 # barrier
+    # crossover: 0
+    # BarConvTol: 1.e-6
+    # Seed: 123
+    # AggFill: 0
+    # PreDual: 0
+    # GURO_PAR_BARDENSETHRESH: 200
+    #FeasibilityTol: 1.e-6
+
+    #name: cplex
+    #threads: 4
+    #lpmethod: 4 # barrier
+    #solutiontype: 2 # non basic solution, ie no crossover
+    #barrier_convergetol: 1.e-5
+    #feasopt_tolerance: 1.e-6
+  mem: 4000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+
+
+plotting:
+  map:
+    boundaries: [-11, 30, 34, 71]
+    color_geomap:
+      ocean: white
+      land: whitesmoke
+  costs_max: 1000
+  costs_threshold: 1
+  energy_max: 20000
+  energy_min: -20000
+  energy_threshold: 50
+  vre_techs:
+    - onwind
+    - offwind-ac
+    - offwind-dc
+    - solar
+    - ror
+  renewable_storage_techs:
+    - PHS
+    - hydro
+  conv_techs:
+    - OCGT
+    - CCGT
+    - Nuclear
+    - Coal
+  storage_techs:
+    - hydro+PHS
+    - battery
+    - H2
+  load_carriers:
+    - AC load
+  AC_carriers:
+    - AC line
+    - AC transformer
+  link_carriers:
+    - DC line
+    - Converter AC-DC
+  heat_links:
+    - heat pump
+    - resistive heater
+    - CHP heat
+    - CHP electric
+    - gas boiler
+    - central heat pump
+    - central resistive heater
+    - central CHP heat
+    - central CHP electric
+    - central gas boiler
+  heat_generators:
+    - gas boiler
+    - central gas boiler
+    - solar thermal collector
+    - central solar thermal collector
+  tech_colors:
+    # wind
+    onwind: "#235ebc"
+    onshore wind: "#235ebc"
+    offwind: "#6895dd"
+    offshore wind: "#6895dd"
+    offwind-ac: "#6895dd"
+    offshore wind (AC): "#6895dd"
+    offwind-dc: "#74c6f2"
+    offshore wind (DC): "#74c6f2"
+    # water
+    hydro: '#298c81'
+    hydro reservoir: '#298c81'
+    ror: '#3dbfb0'
+    run of river: '#3dbfb0'
+    hydroelectricity: '#298c81'
+    PHS: '#51dbcc'
+    wave: '#a7d4cf'
+    # solar
+    solar: "#f9d002"
+    solar PV: "#f9d002"
+    solar thermal: '#ffbf2b'
+    solar rooftop: '#ffea80'
+    # gas
+    OCGT: '#e0986c'
+    OCGT marginal: '#e0986c'
+    OCGT-heat: '#e0986c'
+    gas boiler: '#db6a25'
+    gas boilers: '#db6a25'
+    gas boiler marginal: '#db6a25'
+    gas: '#e05b09'
+    fossil gas: '#e05b09'
+    natural gas: '#e05b09'
+    CCGT: '#a85522'
+    CCGT marginal: '#a85522'
+    gas for industry co2 to atmosphere: '#692e0a'
+    gas for industry co2 to stored: '#8a3400'
+    gas for industry: '#853403'
+    gas for industry CC: '#692e0a'
+    gas pipeline: '#ebbca0'
+    gas pipeline new: '#a87c62'
+    # oil
+    oil: '#c9c9c9'
+    oil boiler: '#adadad'
+    agriculture machinery oil: '#949494'
+    shipping oil: "#808080"
+    land transport oil: '#afafaf'
+    # nuclear
+    Nuclear: '#ff8c00'
+    Nuclear marginal: '#ff8c00'
+    nuclear: '#ff8c00'
+    uranium: '#ff8c00'
+    # coal
+    Coal: '#545454'
+    coal: '#545454'
+    Coal marginal: '#545454'
+    solid: '#545454'
+    Lignite: '#826837'
+    lignite: '#826837'
+    Lignite marginal: '#826837'
+    # biomass
+    biogas: '#e3d37d'
+    biomass: '#baa741'
+    solid biomass: '#baa741'
+    solid biomass transport: '#baa741'
+    solid biomass for industry: '#7a6d26'
+    solid biomass for industry CC: '#47411c'
+    solid biomass for industry co2 from atmosphere: '#736412'
+    solid biomass for industry co2 to stored: '#47411c'
+    # power transmission
+    lines: '#6c9459'
+    transmission lines: '#6c9459'
+    electricity distribution grid: '#97ad8c'
+    # electricity demand
+    Electric load: '#110d63'
+    electric demand: '#110d63'
+    electricity: '#110d63'
+    industry electricity: '#2d2a66'
+    industry new electricity: '#2d2a66'
+    agriculture electricity: '#494778'
+    # battery + EVs
+    battery: '#ace37f'
+    battery storage: '#ace37f'
+    home battery: '#80c944'
+    home battery storage: '#80c944'
+    BEV charger: '#baf238'
+    V2G: '#e5ffa8'
+    land transport EV: '#baf238'
+    Li ion: '#baf238'
+    # hot water storage
+    water tanks: '#e69487'
+    hot water storage: '#e69487'
+    hot water charging: '#e69487'
+    hot water discharging: '#e69487'
+    # heat demand
+    Heat load: '#cc1f1f'
+    heat: '#cc1f1f'
+    heat demand: '#cc1f1f'
+    rural heat: '#ff5c5c'
+    central heat: '#cc1f1f'
+    decentral heat: '#750606'
+    low-temperature heat for industry: '#8f2727'
+    process heat: '#ff0000'
+    agriculture heat: '#d9a5a5'
+    # heat supply
+    heat pumps: '#2fb537'
+    heat pump: '#2fb537'
+    air heat pump: '#36eb41'
+    ground heat pump: '#2fb537'
+    Ambient: '#98eb9d'
+    CHP: '#8a5751'
+    CHP CC: '#634643'
+    CHP heat: '#8a5751'
+    CHP electric: '#8a5751'
+    district heating: '#e8beac'
+    resistive heater: '#d8f9b8'
+    retrofitting: '#8487e8'
+    building retrofitting: '#8487e8'
+    # hydrogen
+    H2 for industry: "#f073da"
+    H2 for shipping: "#ebaee0"
+    H2: '#bf13a0'
+    hydrogen: '#bf13a0'
+    SMR: '#870c71'
+    SMR CC: '#4f1745'
+    H2 liquefaction: '#d647bd'
+    hydrogen storage: '#bf13a0'
+    H2 storage: '#bf13a0'
+    land transport fuel cell: '#6b3161'
+    H2 pipeline: '#f081dc'
+    H2 pipeline retrofitted: '#ba99b5'
+    H2 Fuel Cell: '#c251ae'
+    H2 Electrolysis: '#ff29d9'
+    # syngas
+    Sabatier: '#9850ad'
+    methanation: '#c44ce6'
+    methane: '#c44ce6'
+    helmeth: '#e899ff'
+    # synfuels
+    Fischer-Tropsch: '#25c49a'
+    liquid: '#25c49a'
+    kerosene for aviation: '#a1ffe6'
+    naphtha for industry: '#57ebc4'
+    # co2
+    CC: '#f29dae'
+    CCS: '#f29dae'
+    CO2 sequestration: '#f29dae'
+    DAC: '#ff5270'
+    co2 stored: '#f2385a'
+    co2: '#f29dae'
+    co2 vent: '#ffd4dc'
+    CO2 pipeline: '#f5627f'
+    # emissions
+    process emissions CC: '#000000'
+    process emissions: '#222222'
+    process emissions to stored: '#444444'
+    process emissions to atmosphere: '#888888'
+    oil emissions: '#aaaaaa'
+    shipping oil emissions: "#555555"
+    land transport oil emissions: '#777777'
+    agriculture machinery oil emissions: '#333333'
+    # other
+    shipping: '#03a2ff'
+    power-to-heat: '#2fb537'
+    power-to-gas: '#c44ce6'
+    power-to-H2: '#ff29d9'
+    power-to-liquid: '#25c49a'
+    gas-to-power/heat: '#ee8340'
+    waste: '#e3d37d'
+    other: '#000000'

From 31330bbf4b1c22d27e142a37725339e312e1446c Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:27:13 +0200
Subject: [PATCH 04/18] fix datafiles list to work with directories

---
 Snakefile | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/Snakefile b/Snakefile
index 329e03e9..2ecff903 100644
--- a/Snakefile
+++ b/Snakefile
@@ -45,22 +45,22 @@ rule prepare_sector_networks:
                **config['scenario'])
 
 datafiles = [
-    "eea/UNFCCC_v23.csv",
-    "switzerland-sfoe/switzerland-new_format.csv",
-    "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
-    "myb1-2017-nitro.xls",
-    "Industrial_Database.csv",
-    "emobility/KFZ__count",
-    "emobility/Pkw__count",
-    "eurostat-energy_balances-june_2016_edition",
-    "eurostat-energy_balances-may_2018_edition",
-    "jrc-idees-2015",
-    "h2_salt_caverns_GWh_per_sqkm.geojson",
+    "data/eea/UNFCCC_v23.csv",
+    "data/switzerland-sfoe/switzerland-new_format.csv",
+    "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson",
+    "data/myb1-2017-nitro.xls",
+    "data/Industrial_Database.csv",
+    "data/emobility/KFZ__count",
+    "data/emobility/Pkw__count",
+    "data/h2_salt_caverns_GWh_per_sqkm.geojson",
+    directory("data/eurostat-energy_balances-june_2016_edition"),
+    directory("data/eurostat-energy_balances-may_2018_edition"),
+    directory("data/jrc-idees-2015"),
 ]
 
 if config.get('retrieve_sector_databundle', True):
     rule retrieve_sector_databundle:
-        output:  expand('data/{file}', file=datafiles)
+        output: *datafiles
         log: "logs/retrieve_sector_databundle.log"
         script: 'scripts/retrieve_sector_databundle.py'
 

From 941b7ba4dbc02ed75676bdd40c755c55d570f76c Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:27:43 +0200
Subject: [PATCH 05/18] prepare: only add new gas pipes if augmentation not
 empty

---
 scripts/prepare_sector_network.py | 33 +++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 9ad969fb..6cdaeddd 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1155,23 +1155,26 @@ def add_storage_and_grids(n, costs):
         # apply k_edge_augmentation weighted by length of complement edges
         k_edge = options.get("gas_network_connectivity_upgrade", 3)
         augmentation = k_edge_augmentation(G, k_edge, avail=complement_edges.values)
-        new_gas_pipes = pd.DataFrame(augmentation, columns=["bus0", "bus1"])
-        new_gas_pipes["length"] = new_gas_pipes.apply(haversine, axis=1)
 
-        new_gas_pipes.index = new_gas_pipes.apply(
-            lambda x: f"gas pipeline new {x.bus0} <-> {x.bus1}", axis=1)
+        if list(augmentation):
 
-        n.madd("Link",
-            new_gas_pipes.index,
-            bus0=new_gas_pipes.bus0 + " gas",
-            bus1=new_gas_pipes.bus1 + " gas",
-            p_min_pu=-1, # new gas pipes are bidirectional
-            p_nom_extendable=True,
-            length=new_gas_pipes.length,
-            capital_cost=new_gas_pipes.length * costs.at['CH4 (g) pipeline', 'fixed'],
-            carrier="gas pipeline new",
-            lifetime=costs.at['CH4 (g) pipeline', 'lifetime']
-        )
+            new_gas_pipes = pd.DataFrame(augmentation, columns=["bus0", "bus1"])
+            new_gas_pipes["length"] = new_gas_pipes.apply(haversine, axis=1)
+
+            new_gas_pipes.index = new_gas_pipes.apply(
+                lambda x: f"gas pipeline new {x.bus0} <-> {x.bus1}", axis=1)
+
+            n.madd("Link",
+                new_gas_pipes.index,
+                bus0=new_gas_pipes.bus0 + " gas",
+                bus1=new_gas_pipes.bus1 + " gas",
+                p_min_pu=-1, # new gas pipes are bidirectional
+                p_nom_extendable=True,
+                length=new_gas_pipes.length,
+                capital_cost=new_gas_pipes.length * costs.at['CH4 (g) pipeline', 'fixed'],
+                carrier="gas pipeline new",
+                lifetime=costs.at['CH4 (g) pipeline', 'lifetime']
+            )
 
     if options["H2_retrofit"]:
 

From 8201ebcc54740efafd7b665ff6639b899e1f0fcd Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:28:06 +0200
Subject: [PATCH 06/18] prepare: handle case where salt caverns dataframe is
 empty

---
 scripts/prepare_sector_network.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index 6cdaeddd..b1cd5b57 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -1045,18 +1045,20 @@ def add_storage_and_grids(n, costs):
     )
 
     cavern_types = snakemake.config["sector"]["hydrogen_underground_storage_locations"]
-    h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)[cavern_types].sum(axis=1)
-
-    # only use sites with at least 2 TWh potential
-    h2_caverns = h2_caverns[h2_caverns > 2]
+    h2_caverns = pd.read_csv(snakemake.input.h2_cavern, index_col=0)
     
-    # convert TWh to MWh
-    h2_caverns = h2_caverns * 1e6
+    if not h2_caverns.empty and options['hydrogen_underground_storage']:
+
+        h2_caverns = h2_caverns[cavern_types].sum(axis=1)
 
-    # clip at 1000 TWh for one location
-    h2_caverns.clip(upper=1e9, inplace=True)
+        # only use sites with at least 2 TWh potential
+        h2_caverns = h2_caverns[h2_caverns > 2]
+        
+        # convert TWh to MWh
+        h2_caverns = h2_caverns * 1e6
 
-    if options['hydrogen_underground_storage']:
+        # clip at 1000 TWh for one location
+        h2_caverns.clip(upper=1e9, inplace=True)
 
         logger.info("Add hydrogen underground storage")
 

From 952166787e9a02ed842c071c3edb31113d3e34a3 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:28:27 +0200
Subject: [PATCH 07/18] prepare: handle case where underwater faction not in
 link columns

---
 scripts/prepare_sector_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py
index b1cd5b57..56f448fa 100644
--- a/scripts/prepare_sector_network.py
+++ b/scripts/prepare_sector_network.py
@@ -252,6 +252,7 @@ def create_network_topology(n, prefix, carriers=["DC"], connector=" -> ", bidire
 
     ln_attrs = ["bus0", "bus1", "length"]
     lk_attrs = ["bus0", "bus1", "length", "underwater_fraction"]
+    lk_attrs = n.links.columns.intersection(lk_attrs)
 
     candidates = pd.concat([
         n.lines[ln_attrs],

From aba020f9b83ff39b1912584dca0b3bf302f0e89a Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:29:04 +0200
Subject: [PATCH 08/18] build_biomass: handle case where Balkan not in country
 list

---
 scripts/build_biomass_potentials.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py
index 2b6812f3..c9a2594d 100644
--- a/scripts/build_biomass_potentials.py
+++ b/scripts/build_biomass_potentials.py
@@ -144,7 +144,9 @@ def build_nuts2_shapes():
     nuts2 = gpd.GeoDataFrame(gpd.read_file(snakemake.input.nuts2).set_index('id').geometry)
 
     countries = gpd.read_file(snakemake.input.country_shapes).set_index('name')
-    missing = countries.loc[["AL", "RS", "BA"]]
+    missing_iso2 = countries.index.intersection(["AL", "RS", "BA"])
+    missing = countries.loc[missing_iso2]
+
     nuts2.rename(index={"ME00": "ME", "MK00": "MK"}, inplace=True)
 
     return nuts2.append(missing)

From 9ae3ace243d10e03587b81efc85d3c9e36d04699 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:33:24 +0200
Subject: [PATCH 09/18] refer correct environment path

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7e032126..26afb590 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -84,7 +84,7 @@ jobs:
           key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
 
       - name: Update environment due to outdated or unavailable cache
-        run: mamba env update -n pypsa-eur -f envs/environment.yaml
+        run: mamba env update -n pypsa-eur -f ../pypsa-eur/envs/environment.yaml
         if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Test snakemake workflow

From de6fd9b2632579abbf14495ca682f619a66d6560 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:41:43 +0200
Subject: [PATCH 10/18] fix Snakemake call

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 26afb590..d4c359c5 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -91,5 +91,5 @@ jobs:
         run: |
           conda activate pypsa-eur
           conda list
-          snakemake -jall --configfile test/config.overnight.yaml solve_all_networks
-          snakemake -jall --configfile test/config.myopic.yaml solve_all_networks
+          snakemake solve_all_networks -jall --configfile test/config.overnight.yaml 
+          snakemake solve_all_networks -jall --configfile test/config.myopic.yaml

From daf68eb328597d952de95b2a8aeaac13892fd51c Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 11:54:27 +0200
Subject: [PATCH 11/18] make subworkflow dependency explicit again

---
 .github/workflows/ci.yaml | 2 +-
 Snakefile                 | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index d4c359c5..9e80a09d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -81,7 +81,7 @@ jobs:
         id: cache
         with:
           path: ${{ matrix.prefix }}
-          key: ${{ matrix.label }}-conda-${{ hashFiles('envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+          key: ${{ matrix.label }}-conda-${{ hashFiles('../pypsa-eur/envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
 
       - name: Update environment due to outdated or unavailable cache
         run: mamba env update -n pypsa-eur -f ../pypsa-eur/envs/environment.yaml
diff --git a/Snakefile b/Snakefile
index 2ecff903..024c3a4d 100644
--- a/Snakefile
+++ b/Snakefile
@@ -256,9 +256,9 @@ rule build_biomass_potentials:
         enspreso_biomass=HTTP.remote("https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True),
         nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21
         regions_onshore=pypsaeur("resources/regions_onshore_elec_s{simpl}_{clusters}.geojson"),
-        nuts3_population="../pypsa-eur/data/bundle/nama_10r_3popgdp.tsv.gz",
-        swiss_cantons="../pypsa-eur/data/bundle/ch_cantons.csv",
-        swiss_population="../pypsa-eur/data/bundle/je-e-21.03.02.xls",
+        nuts3_population=pypsaeur("data/bundle/nama_10r_3popgdp.tsv.gz"),
+        swiss_cantons=pypsaeur("data/bundle/ch_cantons.csv"),
+        swiss_population=pypsaeur("data/bundle/je-e-21.03.02.xls"),
         country_shapes=pypsaeur('resources/country_shapes.geojson')
     output:
         biomass_potentials_all='resources/biomass_potentials_all_s{simpl}_{clusters}.csv',

From d0eb81cdd8d6d409f1132ffacaf3d6cbf05acea2 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 12:00:47 +0200
Subject: [PATCH 12/18] hashing pypsa-eur environment file does not seem to
 work

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 9e80a09d..b7e4cc91 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -55,7 +55,7 @@ jobs:
         run: |
           git clone https://github.com/pypsa/pypsa-eur ../pypsa-eur
           git clone https://github.com/pypsa/technology-data ../technology-data
-          cp ../pypsa-eur/test/config.test1.yaml  ../pypsa-eur/config.yaml
+          cp ../pypsa-eur/test/config.test1.yaml ../pypsa-eur/config.yaml
       
       - name: Setup secrets
         run: |
@@ -81,7 +81,7 @@ jobs:
         id: cache
         with:
           path: ${{ matrix.prefix }}
-          key: ${{ matrix.label }}-conda-${{ hashFiles('../pypsa-eur/envs/environment.yaml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+          key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
 
       - name: Update environment due to outdated or unavailable cache
         run: mamba env update -n pypsa-eur -f ../pypsa-eur/envs/environment.yaml

From 948dd8f540fc86a2006dcf52f35614e651464c5e Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 12:18:10 +0200
Subject: [PATCH 13/18] change solver to glpk

---
 test/config.myopic.yaml    | 3 ++-
 test/config.overnight.yaml | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml
index ab8b0765..aa22199e 100644
--- a/test/config.myopic.yaml
+++ b/test/config.myopic.yaml
@@ -362,7 +362,8 @@ solving:
       - StorageUnit
 
   solver:
-    name: cbc
+    name: glpk
+    seed: 1
     # threads: 4
     # method: 2 # barrier
     # crossover: 0
diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml
index 3764c8fa..d9e4a5b0 100644
--- a/test/config.overnight.yaml
+++ b/test/config.overnight.yaml
@@ -360,7 +360,8 @@ solving:
       - StorageUnit
 
   solver:
-    name: cbc
+    name: glpk
+    seed: 1
     # threads: 4
     # method: 2 # barrier
     # crossover: 0

From f5cd772c1fc41586bb5ed46a444e1ccef79aadd2 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 12:18:19 +0200
Subject: [PATCH 14/18] add data cache

---
 .github/workflows/ci.yaml | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b7e4cc91..fcee60c4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -19,7 +19,8 @@ on:
     - cron: "0 5 * * TUE"
 
 env:
-  CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  CONDA_CACHE_NUMBER: 1 # Change this value to manually reset the environment cache
+  DATA_CACHE_NUMBER: 1
 
 jobs:
   build:
@@ -73,15 +74,26 @@ jobs:
             activate-environment: pypsa-eur
             use-mamba: true
             
-      - name: Set cache date
-        run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+      - name: Set cache dates
+        run: |
+          echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
+          echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV
+
+      - name: Cache data and cutouts folders
+        uses: actions/cache@v3
+        with:
+          path: |
+            data
+            ../pypsa-eur/cutouts
+            ../pypsa-eur/data
+          key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }}
 
       - name: Create environment cache
         uses: actions/cache@v2
         id: cache
         with:
           path: ${{ matrix.prefix }}
-          key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
+          key: ${{ matrix.label }}-conda-${{ env.DATE }}-${{ env.CONDA_CACHE_NUMBER }}
 
       - name: Update environment due to outdated or unavailable cache
         run: mamba env update -n pypsa-eur -f ../pypsa-eur/envs/environment.yaml

From 903253c11f0da0ec2b0b21e55b29581d1a9c2661 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 12:35:08 +0200
Subject: [PATCH 15/18] glpk without options

---
 test/config.overnight.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml
index d9e4a5b0..2ba0617f 100644
--- a/test/config.overnight.yaml
+++ b/test/config.overnight.yaml
@@ -361,7 +361,7 @@ solving:
 
   solver:
     name: glpk
-    seed: 1
+    # seed: 1
     # threads: 4
     # method: 2 # barrier
     # crossover: 0

From e77361c9214ffb887fc031bba6d01a6d31b9c3aa Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 12:45:27 +0200
Subject: [PATCH 16/18] try cbc

---
 .github/workflows/ci.yaml  | 2 +-
 test/config.myopic.yaml    | 3 +--
 test/config.overnight.yaml | 3 +--
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index fcee60c4..6fc2c16b 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -64,7 +64,7 @@ jobs:
 
       - name: Add solver to environment
         run: |
-          echo -e "  - glpk\n  - ipopt<3.13.3" >> ../pypsa-eur/envs/environment.yaml
+          echo -e "  - coincbc\n  - ipopt<3.13.3" >> ../pypsa-eur/envs/environment.yaml
 
       - name: Setup Mambaforge
         uses: conda-incubator/setup-miniconda@v2
diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml
index aa22199e..ab8b0765 100644
--- a/test/config.myopic.yaml
+++ b/test/config.myopic.yaml
@@ -362,8 +362,7 @@ solving:
       - StorageUnit
 
   solver:
-    name: glpk
-    seed: 1
+    name: cbc
     # threads: 4
     # method: 2 # barrier
     # crossover: 0
diff --git a/test/config.overnight.yaml b/test/config.overnight.yaml
index 2ba0617f..3764c8fa 100644
--- a/test/config.overnight.yaml
+++ b/test/config.overnight.yaml
@@ -360,8 +360,7 @@ solving:
       - StorageUnit
 
   solver:
-    name: glpk
-    # seed: 1
+    name: cbc
     # threads: 4
     # method: 2 # barrier
     # crossover: 0

From 29d961a121e1abaa94a975f371760a2b0276e6be Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Sun, 3 Apr 2022 13:03:10 +0200
Subject: [PATCH 17/18] alternative snakemake test workflow

---
 .github/workflows/ci.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 6fc2c16b..7d54bd18 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -103,5 +103,7 @@ jobs:
         run: |
           conda activate pypsa-eur
           conda list
-          snakemake solve_all_networks -jall --configfile test/config.overnight.yaml 
-          snakemake solve_all_networks -jall --configfile test/config.myopic.yaml
+          cp test/config.overnight.yaml config.yaml
+          snakemake -call solve_all_networks
+          cp test/config.myopic.yaml config.yaml
+          snakemake -call solve_all_networks

From c961fb70b2da7ce5749265cafaf3d81804747eae Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Mon, 11 Apr 2022 17:07:39 +0200
Subject: [PATCH 18/18] test: add exogenous pathways to config

---
 test/config.myopic.yaml | 110 ++++++++++++++++++++--------------------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/test/config.myopic.yaml b/test/config.myopic.yaml
index ab8b0765..1a54b937 100644
--- a/test/config.myopic.yaml
+++ b/test/config.myopic.yaml
@@ -148,11 +148,11 @@ sector:
     potential: 0.6  # maximum fraction of urban demand which can be supplied by district heating
      # increase of today's district heating demand to potential maximum district heating share
      # progress = 0 means today's district heating share, progress = 1 means maximum fraction of urban demand is supplied by district heating
-    progress: 1
-      # 2020: 0.0
-      # 2030: 0.3
-      # 2040: 0.6
-      # 2050: 1.0
+    progress:
+      2020: 0.0
+      2030: 0.3
+      2040: 0.6
+      2050: 1.0
     district_heating_loss: 0.15
   bev_dsm_restriction_value: 0.75  #Set to 0 for no restriction on BEV DSM
   bev_dsm_restriction_time: 7  #Time at which SOC of BEV has to be dsm_restriction_value
@@ -172,16 +172,16 @@ sector:
   bev_avail_mean: 0.8
   v2g: true #allows feed-in to grid from EV battery
   #what is not EV or FCEV is oil-fuelled ICE
-  land_transport_fuel_cell_share: 0.15 # 1 means all FCEVs
-    # 2020: 0
-    # 2030: 0.05
-    # 2040: 0.1
-    # 2050: 0.15
-  land_transport_electric_share: 0.85 # 1 means all EVs
-    # 2020: 0
-    # 2030: 0.25
-    # 2040: 0.6
-    # 2050: 0.85
+  land_transport_fuel_cell_share:
+    2020: 0
+    2030: 0.05
+    2040: 0.1
+    2050: 0.15
+  land_transport_electric_share:
+    2020: 0
+    2030: 0.25
+    2040: 0.6
+    2050: 0.85
   transport_fuel_cell_efficiency: 0.5
   transport_internal_combustion_efficiency: 0.3
   agriculture_machinery_electric_share: 0
@@ -189,29 +189,29 @@ sector:
   agriculture_machinery_electric_efficiency: 0.3 # electricity per use
   shipping_average_efficiency: 0.4 #For conversion of fuel oil to propulsion in 2011
   shipping_hydrogen_liquefaction: false # whether to consider liquefaction costs for shipping H2 demands
-  shipping_hydrogen_share: 1 # 1 means all hydrogen FC
-    # 2020: 0
-    # 2025: 0
-    # 2030: 0.05
-    # 2035: 0.15
-    # 2040: 0.3
-    # 2045: 0.6
-    # 2050: 1
+  shipping_hydrogen_share:
+    2020: 0
+    2025: 0
+    2030: 0.05
+    2035: 0.15
+    2040: 0.3
+    2045: 0.6
+    2050: 1
   time_dep_hp_cop: true #time dependent heat pump coefficient of performance
   heat_pump_sink_T: 55. # Celsius, based on DTU / large area radiators; used in build_cop_profiles.py
    # conservatively high to cover hot water and space heating in poorly-insulated buildings
   reduce_space_heat_exogenously: true  # reduces space heat demand by a given factor (applied before losses in DH)
   # this can represent e.g. building renovation, building demolition, or if
   # the factor is negative: increasing floor area, increased thermal comfort, population growth
-  reduce_space_heat_exogenously_factor: 0.29 # per unit reduction in space heat demand
+  reduce_space_heat_exogenously_factor: # 0.29 # per unit reduction in space heat demand
   # the default factors are determined by the LTS scenario from http://tool.european-calculator.eu/app/buildings/building-types-area/?levers=1ddd4444421213bdbbbddd44444ffffff11f411111221111211l212221
-    # 2020: 0.10  # this results in a space heat demand reduction of 10%
-    # 2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
-    # 2030: 0.09
-    # 2035: 0.11
-    # 2040: 0.16
-    # 2045: 0.21
-    # 2050: 0.29
+    2020: 0.10  # this results in a space heat demand reduction of 10%
+    2025: 0.09  # first heat demand increases compared to 2020 because of larger floor area per capita
+    2030: 0.09
+    2035: 0.11
+    2040: 0.16
+    2045: 0.21
+    2050: 0.29
   retrofitting :  # co-optimises building renovation to reduce space heat demand
     retro_endogen: false  # co-optimise space heat savings
     cost_factor: 1.0   # weight costs for building renovation
@@ -265,32 +265,32 @@ sector:
 
 
 industry:
-  St_primary_fraction: 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
-    # 2020: 0.6
-    # 2025: 0.55
-    # 2030: 0.5
-    # 2035: 0.45
-    # 2040: 0.4
-    # 2045: 0.35
-    # 2050: 0.3
-  DRI_fraction: 1 # fraction of the primary route converted to DRI + EAF
-    # 2020: 0
-    # 2025: 0
-    # 2030: 0.05
-    # 2035: 0.2
-    # 2040: 0.4
-    # 2045: 0.7
-    # 2050: 1
+  St_primary_fraction: # 0.3 # fraction of steel produced via primary route versus secondary route (scrap+EAF); today fraction is 0.6
+    2020: 0.6
+    2025: 0.55
+    2030: 0.5
+    2035: 0.45
+    2040: 0.4
+    2045: 0.35
+    2050: 0.3
+  DRI_fraction: # 1 # fraction of the primary route converted to DRI + EAF
+    2020: 0
+    2025: 0
+    2030: 0.05
+    2035: 0.2
+    2040: 0.4
+    2045: 0.7
+    2050: 1
   H2_DRI: 1.7   #H2 consumption in Direct Reduced Iron (DRI),  MWh_H2,LHV/ton_Steel from 51kgH2/tSt in Vogl et al (2018) doi:10.1016/j.jclepro.2018.08.279
   elec_DRI: 0.322   #electricity consumption in Direct Reduced Iron (DRI) shaft, MWh/tSt HYBRIT brochure https://ssabwebsitecdn.azureedge.net/-/media/hybrit/files/hybrit_brochure.pdf
-  Al_primary_fraction: 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
-    # 2020: 0.4
-    # 2025: 0.375
-    # 2030: 0.35
-    # 2035: 0.325
-    # 2040: 0.3
-    # 2045: 0.25
-    # 2050: 0.2
+  Al_primary_fraction: # 0.2 # fraction of aluminium produced via the primary route versus scrap; today fraction is 0.4
+    2020: 0.4
+    2025: 0.375
+    2030: 0.35
+    2035: 0.325
+    2040: 0.3
+    2045: 0.25
+    2050: 0.2
   MWh_CH4_per_tNH3_SMR: 10.8 # 2012's demand from https://ec.europa.eu/docsroom/documents/4165/attachments/1/translations/en/renditions/pdf
   MWh_elec_per_tNH3_SMR: 0.7 # same source, assuming 94-6% split methane-elec of total energy demand 11.5 MWh/tNH3
   MWh_H2_per_tNH3_electrolysis: 6.5 # from https://doi.org/10.1016/j.joule.2018.04.017, around 0.197 tH2/tHN3 (>3/17 since some H2 lost and used for energy)