Merge pull request #409 from calliope-project/cleanup-templates

CHANGELOG.md

@@ -58,7 +58,7 @@
 * **UPDATED** structure of YAML templates and parametrisation:
     * Parametrisation moved to eurocalliopelib.
     * Rules to parametrise split into smaller technology-specific rules, to ensure inputs are directly relevant to the files being parametrised.
-    * YAML templates restructured to match structure of final model (see `Updated (models) above`);
+    * YAML templates restructured to match structure of final model (see `Updated (models) above`) and given `.jinja` suffix to allow for IDE syntax highlighting (#404);
 * **UPDATE** cluster sync infrastructure to retain file permission defaults on the cluster. This change improves team collaboration, as default group settings will apply to the files on the cluster (#214).
 * **UPDATE** the declaration of required cluster resources. Moving away from a mechanism that is deprecated in Snakemake (#211).
 * **UPDATE** default Snakemake profile to be activated automatically, for convenience (#264, #268).

Snakefile

@@ -30,11 +30,9 @@ include: "./rules/modules.smk"
 min_version("8.10")
 localrules: all, clean
 wildcard_constraints:
-    resolution = "continental|national|regional|ehighways"
-
-ruleorder: area_to_capacity_limits > hydro_capacities > biofuels > nuclear_regional_capacity > dummy_tech_locations_template
-ruleorder: bio_techs_and_locations_template > techs_and_locations_template
-ruleorder: create_controlled_road_transport_annual_demand_and_installed_capacities > dummy_tech_locations_template
+    resolution = "continental|national|regional|ehighways",
+    group_and_tech = "(demand|storage|supply|transmission)\/\w+"
+ruleorder: module_with_location_specific_data > module_without_location_specific_data
 
 ALL_CF_TECHNOLOGIES = [
     "wind-onshore", "wind-offshore", "open-field-pv",
@@ -96,32 +94,30 @@ rule all_tests:
         )
 
 
-rule dummy_tech_locations_template:  # needed to provide `techs_and_locations_template` with a locational CSV linked to each technology that has no location-specific data to define.
-    message: "Create empty {wildcards.resolution} location-specific data file for the {wildcards.tech_group} tech `{wildcards.tech}`."  # Update ruleorder at the top of the file if you instead want the techs_and_locations_template rule to be used to generate a file
-    input: rules.locations_template.output.csv
-    output: "build/data/{resolution}/{tech_group}/{tech}.csv"
-    conda: "envs/shell.yaml"
-    shell: "cp {input} {output}"
-
-
-rule techs_and_locations_template:
-    message: "Create {wildcards.resolution} definition file for the {wildcards.tech_group} tech `{wildcards.tech}`."
+rule module_with_location_specific_data:
+    message: "Create {wildcards.resolution} definition file for {wildcards.group_and_tech}."
     input:
-        template = techs_template_dir + "{tech_group}/{tech}.yaml",
-        locations = "build/data/{resolution}/{tech_group}/{tech}.csv"
+        template = techs_template_dir + "{group_and_tech}.yaml.jinja",
+        locations = "build/data/{resolution}/{group_and_tech}.csv"
     params:
         scaling_factors = config["scaling-factors"],
         capacity_factors = config["capacity-factors"]["average"],
         max_power_densities = config["parameters"]["maximum-installable-power-density"],
         heat_pump_shares = config["parameters"]["heat-pump"]["heat-pump-shares"],
     wildcard_constraints:
-        tech_group = "(?!transmission).*"  # i.e. all but transmission
+        # Exclude all outputs that have their own `techs_and_locations_template` implementation
+        group_and_tech = "(?!transmission\/|supply\/biofuel).*"
     conda: "envs/default.yaml"
-    output: "build/models/{resolution}/techs/{tech_group}/{tech}.yaml"
+    output: "build/models/{resolution}/techs/{group_and_tech}.yaml"
     script: "scripts/template_techs.py"
 
+use rule module_with_location_specific_data as module_without_location_specific_data with:
+    # For all cases where we don't have any location-specific data that we want to supply to the template
+    input:
+        template = techs_template_dir + "{group_and_tech}.yaml.jinja",
+        locations = rules.locations_module.output.csv
 
-rule no_params_model_template:
+rule module_without_specific_data:
     message: "Create {wildcards.resolution} configuration files from templates where no parameterisation is required."
     input:
         template = model_template_dir + "{template}",
@@ -132,8 +128,8 @@ rule no_params_model_template:
     shell: "cp {input.template} {output}"
 
 
-rule no_params_template:
-    message: "Create non-model files from templates where no parameterisation is required."
+rule auxiliary_files:
+    message: "Create auxiliary output files (i.e. those not used to define a Calliope model) from templates where no parameterisation is required."
     input:
         template = template_dir + "{template}",
     output: "build/models/{template}"
@@ -143,16 +139,16 @@ rule no_params_template:
     shell: "cp {input.template} {output}"
 
 
-rule model_template:
+rule model:
     message: "Generate top-level {wildcards.resolution} model configuration file from template"
     input:
-        template = model_template_dir + "example-model.yaml",
-        non_model_files = expand(
+        template = model_template_dir + "example-model.yaml.jinja",
+        auxiliary_files = expand(
             "build/models/{template}", template=["environment.yaml", "README.md"]
         ),
-        input_files = expand(
-            "build/models/{{resolution}}/{input_file}",
-            input_file=[
+        modules = expand(
+            "build/models/{{resolution}}/{module}",
+            module=[
                 "interest-rate.yaml",
                 "locations.yaml",
                 "techs/demand/electricity.yaml",
@@ -187,9 +183,9 @@ rule model_template:
             "build/models/{resolution}/timeseries/demand/demand-shape-equals-ev.csv",
             "build/models/{resolution}/timeseries/demand/plugin-profiles-ev.csv",
         ),
-        optional_input_files = lambda wildcards: expand(
-            f"build/models/{wildcards.resolution}/{{input_file}}",
-            input_file=[
+        optional_modules = lambda wildcards: expand(
+            f"build/models/{wildcards.resolution}/{{module}}",
+            module=[
                 "techs/transmission/electricity-linked-neighbours.yaml",
             ] + ["techs/transmission/electricity-entsoe.yaml" for i in [None] if wildcards.resolution == "national"]
         )

rules/biofuels.smk

@@ -1,6 +1,5 @@
 """Rules related to biofuels."""
 
-
 rule download_biofuel_potentials_and_costs:
     message: "Download raw biofuel potential and cost data."
     params: url = config["data-sources"]["biofuel-potentials-and-costs"]
@@ -52,10 +51,10 @@ rule biofuels:
     script: "../scripts/biofuels/allocate.py"
 
 
-rule bio_techs_and_locations_template:
+rule biofuel_tech_module:
     message: "Create biofuel tech definition file from template."
     input:
-        template = techs_template_dir + "supply/biofuel.yaml",
+        template = techs_template_dir + "supply/biofuel.yaml.jinja",
         biofuel_cost = "build/data/regional/biofuel/{scenario}/costs-eur-per-mwh.csv".format(
             scenario=config["parameters"]["jrc-biofuel"]["scenario"]
         ),

rules/shapes.smk

@@ -140,10 +140,10 @@ rule eez:
         """
 
 
-rule locations_template:
+rule locations_module:
     message: "Generate locations configuration file for {wildcards.resolution} resolution from template."
     input:
-        template = model_template_dir + "locations.yaml",
+        template = model_template_dir + "locations.yaml.jinja",
         shapes = rules.units.output[0]
     output:
         yaml = "build/models/{resolution}/locations.yaml",

rules/transmission.smk

@@ -20,11 +20,11 @@ rule entsoe_tyndp_xlsx:
     shell: "unzip -o {input} 'TYNDP-2020-Scenario-Datafile.xlsx' -d build/data/national"
 
 
-rule transmission_entsoe_tyndp_template:
+rule transmission_entsoe_tyndp_tech_module:
     message: "Create YAML file of national-scale links with ENTSO-E TYNDP net-transfer capacities"
     input:
-        template = techs_template_dir + "transmission/electricity-transmission.yaml",
-        locations = rules.locations_template.output.csv,
+        template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja",
+        locations = rules.locations_module.output.csv,
         entsoe_tyndp = rules.entsoe_tyndp_xlsx.output[0]
     params:
         scenario = config["parameters"]["entsoe-tyndp"]["scenario"],
@@ -39,10 +39,10 @@ rule transmission_entsoe_tyndp_template:
     script: "../scripts/transmission/template_transmission_entsoe_tyndp.py"
 
 
-rule link_locations_with_transmission_techs_template:
+rule transmission_linked_neighbours_tech_module:
     message: "Link {wildcards.resolution} direct neighbours and neighbours with sea connections with transmission techs from template."
     input:
-        template = techs_template_dir + "transmission/electricity-transmission.yaml",
+        template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja",
         units = rules.units.output[0]
     params:
         scaling_factors = config["scaling-factors"],

scripts/template_model.py

@@ -5,8 +5,8 @@
 from eurocalliopelib.template import parametrise_template
 
 
-def construct_model(path_to_template, path_to_output, input_files, resolution, year):
-    input_files = sorted([update_path(file, resolution) for file in input_files])
+def construct_model(path_to_template, path_to_output, modules, resolution, year):
+    input_files = sorted([update_path(file, resolution) for file in modules])
 
     return parametrise_template(
         path_to_template,
@@ -26,7 +26,7 @@ def update_path(path_string, resolution):
 if __name__ == "__main__":
     construct_model(
         path_to_template=snakemake.input.template,
-        input_files=snakemake.input.input_files,
+        modules=snakemake.input.modules,
         resolution=snakemake.wildcards.resolution,
         year=snakemake.params.year,
         path_to_output=snakemake.output[0],