Merge pull request #947 from PyPSA/energy_totals

Eurostat 2023 data for energy totals

.github/workflows/ci.yaml

@@ -19,7 +19,7 @@ on:
   - cron: "0 5 * * TUE"
 
 env:
-  DATA_CACHE_NUMBER: 2
+  DATA_CACHE_NUMBER: 1
 
 jobs:
   build:

config/config.default.yaml

@@ -314,9 +314,8 @@ pypsa_eur:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy
 energy:
-  energy_totals_year: 2013
+  energy_totals_year: 2019
   base_emissions_year: 1990
-  eurostat_report_year: 2016
   emissions: CO2
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass

data/switzerland-new_format-all_years.csv

@@ -1,25 +1,25 @@
-country,item,2010,2011,2012,2013,2014,2015
-CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1
-CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4
-CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9
-CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6
-CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4
-CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5
-CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6
-CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0
-CH,total services,145.9,127.4,136.7,144.0,124.5,132.5
-CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3
-CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0
-CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3
-CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6
-CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6
-CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7
-CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3
-CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4
-CH,total road,199.4,200.4,200.4,201.2,202.0,203.1
-CH,electricity road,0.,0.,0.,0.,0.,0.
-CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4
-CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5
-CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8
-CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6
-CH,total international navigation,0.,0.,0.,0.,0.,0.
+country,item,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022
+CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1,241.2,236.5,223.7,226.5,219.1,241.2,211.3
+CH,total residential space,192.2,149,168.1,185.5,139.7,154.4,167.3,161.5,147.2,150.4,140.2,166.2,131.9
+CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9,31.8,31.8,31.8,31.7,33.3,32.5,32.5
+CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6,9.9,10,10.1,10.2,10.5,10.3,10.3
+CH,electricity residential,67.9,63.7,65.7,67.6,63,64.4,69.7,69.2,67.7,68.1,68.7,70.8,66.8
+CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5,15.8,15.6,14.7,15.3,14.8,17.8,14.8
+CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6,8.9,9,9.2,9.3,9.7,9.5,9.5
+CH,electricity residential cooking,4.9,4.9,4.9,4.9,5,5,5,5.1,5.1,5.1,5.4,5.2,5.3
+CH,total services,145.9,127.4,136.7,144,124.5,132.5,150.5,147.7,141.5,143.1,129.7,144.2,122.5
+CH,total services space,80,62.2,70.8,77.4,58.3,64.3,77,74.4,68.2,69.8,64.3,75.7,58.7
+CH,total services water,10.1,10,10.1,10.1,10,10,11.4,11.3,11.2,11.1,9.7,10.4,12
+CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.2,3.3,2.1,2.6,3.2
+CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6,65.9,65.7,65.5,65.6,58.8,61.6,61.6
+CH,electricity services space,4,3.2,3.8,4.2,3.3,3.6,2.7,2.5,2.3,2.3,2.2,2.5,2.5
+CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7,1.2,1.1,1.1,1.1,0.9,1,1
+CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3,3.1,3.1,3.1,3.2,3.3,2.1,3.2
+CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4,11.6,11.4,11.2,11,10.2,10.6,10.8
+CH,total road,199.4,200.4,200.4,201.2,202,203.1,203.9,203.7,202.6,200.5,182.6,188.3,193.3
+CH,electricity road,0,0,0,0,0,0,0.1,0.2,0.3,0.4,0.5,0.8,1.3
+CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4,11.5,11.3,11.1,11,10.1,10.6,10.7
+CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5,3.6,3.1,3.1,2.9,2.5,2.8,3
+CH,total international aviation,58,62,63.5,64.2,64.5,66.8,70.6,72.8,77.2,78.2,28.2,31.2,56.8
+CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6,1.4,1.4,1.4,1.4,1.4,1.4,1.4
+CH,total international navigation,0,0,0,0,0,0,0,0,0,0,0,0,0

doc/configtables/energy.csv

@@ -1,7 +1,4 @@
 ,Unit,Values,Description
 energy_totals_year ,--,"{1990,1995,2000,2005,2010,2011,…} ",The year for the sector energy use. The year must be avaliable in the Eurostat report
 base_emissions_year ,--,"YYYY; e.g. 1990","The base year for the sector emissions. See `European Environment Agency (EEA) <https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16>`_."
-
-eurostat_report_year ,--,"{2016,2017,2018}","The publication year of the Eurostat report. 2016 includes Bosnia and Herzegovina, 2017 does not"
-
 emissions ,--,"{CO2, All greenhouse gases - (CO2 equivalent)}","Specify which sectoral emissions are taken into account. Data derived from EEA. Currently only CO2 is implemented."

doc/release_notes.rst

@@ -9,6 +9,19 @@ Release Notes
 
 Upcoming Release
 ================
+
+* The Eurostat data was updated to the 2023 version in :mod:`build_energy_totals`.
+
+* The latest `Swiss energy totals
+  <https://www.bfe.admin.ch/bfe/de/home/versorgung/statistik-und-geodaten/energiestatistiken/energieverbrauch-nach-verwendungszweck.html/>`_
+  have been updated to the 2023 version.
+
+* The JRC-IDEES data is only available until 2015. For energy totals years (``energy: energy_totals_year``) after
+  2015, the data scaled using the ratio of Eurostat data reported for the energy
+  totals year and 2015.
+
+* The default energy totals year (``energy: energy_totals_year``) was updated to 2019.
+
 * Upgrade default techno-economic assumptions to ``technology-data`` v0.8.1.
 
 * Linearly interpolate missing investment periods in year-dependent

rules/build_sector.smk

@@ -270,7 +270,7 @@ rule build_energy_totals:
         swiss="data/switzerland-new_format-all_years.csv",
         idees="data/bundle-sector/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
     output:
         energy_name=resources("energy_totals.csv"),
         co2_name=resources("co2_totals.csv"),
@@ -468,7 +468,7 @@ rule build_industrial_production_per_country:
     input:
         ammonia_production=resources("ammonia_production.csv"),
         jrc="data/bundle-sector/jrc-idees-2015",
-        eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition",
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
     output:
         industrial_production_per_country=resources(
             "industrial_production_per_country.csv"
@@ -834,7 +834,6 @@ rule prepare_sector_network:
         countries=config_provider("countries"),
         adjustments=config_provider("adjustments", "sector"),
         emissions_scope=config_provider("energy", "emissions"),
-        eurostat_report_year=config_provider("energy", "eurostat_report_year"),
         RDIR=RDIR,
     input:
         unpack(input_profile_offwind),
@@ -865,7 +864,7 @@ rule prepare_sector_network:
         ),
         network=resources("networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc"),
         energy_totals_name=resources("energy_totals.csv"),
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
         pop_weighted_energy_totals=resources(
             "pop_weighted_energy_totals_s{simpl}_{clusters}.csv"
         ),

rules/common.smk

@@ -130,12 +130,6 @@ def has_internet_access(url="www.zenodo.org") -> bool:
         conn.close()
 
 
-def input_eurostat(w):
-    # 2016 includes BA, 2017 does not
-    report_year = config_provider("energy", "eurostat_report_year")(w)
-    return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition"
-
-
 def solved_previous_horizon(w):
     planning_horizons = config_provider("scenario", "planning_horizons")(w)
     i = planning_horizons.index(int(w.planning_horizons))

rules/postprocess.smk

@@ -237,7 +237,6 @@ rule plot_summary:
         countries=config_provider("countries"),
         planning_horizons=config_provider("scenario", "planning_horizons"),
         emissions_scope=config_provider("energy", "emissions"),
-        eurostat_report_year=config_provider("energy", "eurostat_report_year"),
         plotting=config_provider("plotting"),
         foresight=config_provider("foresight"),
         co2_budget=config_provider("co2_budget"),
@@ -247,7 +246,7 @@ rule plot_summary:
         costs=RESULTS + "csvs/costs.csv",
         energy=RESULTS + "csvs/energy.csv",
         balances=RESULTS + "csvs/supply_energy.csv",
-        eurostat=input_eurostat,
+        eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
         co2="data/bundle-sector/eea/UNFCCC_v23.csv",
     output:
         costs=RESULTS + "graphs/costs.pdf",

rules/retrieve.smk

@@ -135,20 +135,10 @@ if config["enable"]["retrieve"] and config["enable"].get(
         "h2_salt_caverns_GWh_per_sqkm.geojson",
     ]
 
-    datafolders = [
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")
-        ),
-        protected(
-            directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")
-        ),
-        protected(directory("data/bundle-sector/jrc-idees-2015")),
-    ]
-
     rule retrieve_sector_databundle:
         output:
             protected(expand("data/bundle-sector/{files}", files=datafiles)),
-            *datafolders,
+            protected(directory("data/bundle-sector/jrc-idees-2015")),
         log:
             "logs/retrieve_sector_databundle.log",
         retries: 2
@@ -157,6 +147,15 @@ if config["enable"]["retrieve"] and config["enable"].get(
         script:
             "../scripts/retrieve_sector_databundle.py"
 
+    rule retrieve_eurostat_data:
+        output:
+            directory("data/eurostat/eurostat-energy_balances-april_2023_edition"),
+        log:
+            "logs/retrieve_eurostat_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_eurostat_data.py"
+
 
 if config["enable"]["retrieve"]:
     datafiles = [