Add .jinja suffix; reorganise template rules

CHANGELOG.md

@@ -58,7 +58,7 @@
 * **UPDATED** structure of YAML templates and parametrisation:
     * Parametrisation moved to eurocalliopelib.
     * Rules to parametrise split into smaller technology-specific rules, to ensure inputs are directly relevant to the files being parametrised.
-    * YAML templates restructured to match structure of final model (see `Updated (models) above`);
+    * YAML templates restructured to match structure of final model (see `Updated (models) above`) and given `.jinja` suffix to allow for IDE syntax highlighting (#404);
 * **UPDATE** cluster sync infrastructure to retain file permission defaults on the cluster. This change improves team collaboration, as default group settings will apply to the files on the cluster (#214).
 * **UPDATE** the declaration of required cluster resources. Moving away from a mechanism that is deprecated in Snakemake (#211).
 * **UPDATE** default Snakemake profile to be activated automatically, for convenience (#264, #268).

Snakefile

@@ -32,9 +32,7 @@ localrules: all, clean
 wildcard_constraints:
     resolution = "continental|national|regional|ehighways"
 
-ruleorder: area_to_capacity_limits > hydro_capacities > biofuels > nuclear_regional_capacity > dummy_tech_locations_template
-ruleorder: bio_techs_and_locations_template > techs_and_locations_template
-ruleorder: create_controlled_road_transport_annual_demand_and_installed_capacities > dummy_tech_locations_template
+ruleorder: techs_and_locations_template > dummy_tech_locations_template
 
 ALL_CF_TECHNOLOGIES = [
     "wind-onshore", "wind-offshore", "open-field-pv",
@@ -96,30 +94,28 @@ rule all_tests:
         )
 
 
-rule dummy_tech_locations_template:  # needed to provide `techs_and_locations_template` with a locational CSV linked to each technology that has no location-specific data to define.
-    message: "Create empty {wildcards.resolution} location-specific data file for the {wildcards.tech_group} tech `{wildcards.tech}`."  # Update ruleorder at the top of the file if you instead want the techs_and_locations_template rule to be used to generate a file
-    input: rules.locations_template.output.csv
-    output: "build/data/{resolution}/{tech_group}/{tech}.csv"
-    conda: "envs/shell.yaml"
-    shell: "cp {input} {output}"
-
-
 rule techs_and_locations_template:
-    message: "Create {wildcards.resolution} definition file for the {wildcards.tech_group} tech `{wildcards.tech}`."
+    message: "Create {wildcards.resolution} definition file for {wildcards.tech_and_group}."
     input:
-        template = techs_template_dir + "{tech_group}/{tech}.yaml",
-        locations = "build/data/{resolution}/{tech_group}/{tech}.csv"
+        template = techs_template_dir + "{tech_and_group}.yaml.jinja",
+        locations = "build/data/{resolution}/{tech_and_group}.csv"
     params:
         scaling_factors = config["scaling-factors"],
         capacity_factors = config["capacity-factors"]["average"],
         max_power_densities = config["parameters"]["maximum-installable-power-density"],
         heat_pump_shares = config["parameters"]["heat-pump"]["heat-pump-shares"],
     wildcard_constraints:
-        tech_group = "(?!transmission).*"  # i.e. all but transmission
+        # Exclude all outputs that have their own `techs_and_locations_template` implementation
+        tech_and_group = "(?!transmission\/|supply\/biofuel).*"
     conda: "envs/default.yaml"
-    output: "build/models/{resolution}/techs/{tech_group}/{tech}.yaml"
+    output: "build/models/{resolution}/techs/{tech_and_group}.yaml"
     script: "scripts/template_techs.py"
 
+use rule techs_and_locations_template as dummy_tech_locations_template with:
+    # For all cases where we don't have any location-specific data that we want to supply to the template
+    input:
+        template = techs_template_dir + "{tech_and_group}.yaml.jinja",
+        locations = rules.locations_template.output.csv
 
 rule no_params_model_template:
     message: "Create {wildcards.resolution} configuration files from templates where no parameterisation is required."
@@ -146,7 +142,7 @@ rule no_params_template:
 rule model_template:
     message: "Generate top-level {wildcards.resolution} model configuration file from template"
     input:
-        template = model_template_dir + "example-model.yaml",
+        template = model_template_dir + "example-model.yaml.jinja",
         non_model_files = expand(
             "build/models/{template}", template=["environment.yaml", "README.md"]
         ),

rules/biofuels.smk

@@ -1,6 +1,5 @@
 """Rules related to biofuels."""
 
-
 rule download_biofuel_potentials_and_costs:
     message: "Download raw biofuel potential and cost data."
     params: url = config["data-sources"]["biofuel-potentials-and-costs"]
@@ -55,7 +54,7 @@ rule biofuels:
 rule bio_techs_and_locations_template:
     message: "Create biofuel tech definition file from template."
     input:
-        template = techs_template_dir + "supply/biofuel.yaml",
+        template = techs_template_dir + "supply/biofuel.yaml.jinja",
         biofuel_cost = "build/data/regional/biofuel/{scenario}/costs-eur-per-mwh.csv".format(
             scenario=config["parameters"]["jrc-biofuel"]["scenario"]
         ),

rules/shapes.smk

@@ -143,7 +143,7 @@ rule eez:
 rule locations_template:
     message: "Generate locations configuration file for {wildcards.resolution} resolution from template."
     input:
-        template = model_template_dir + "locations.yaml",
+        template = model_template_dir + "locations.yaml.jinja",
         shapes = rules.units.output[0]
     output:
         yaml = "build/models/{resolution}/locations.yaml",

rules/transmission.smk

@@ -20,10 +20,10 @@ rule entsoe_tyndp_xlsx:
     shell: "unzip -o {input} 'TYNDP-2020-Scenario-Datafile.xlsx' -d build/data/national"
 
 
-rule transmission_entsoe_tyndp_template:
+rule transmission_entsoe_tyndp_techs_and_locations_template:
     message: "Create YAML file of national-scale links with ENTSO-E TYNDP net-transfer capacities"
     input:
-        template = techs_template_dir + "transmission/electricity-transmission.yaml",
+        template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja",
         locations = rules.locations_template.output.csv,
         entsoe_tyndp = rules.entsoe_tyndp_xlsx.output[0]
     params:
@@ -39,10 +39,10 @@ rule transmission_entsoe_tyndp_template:
     script: "../scripts/transmission/template_transmission_entsoe_tyndp.py"
 
 
-rule link_locations_with_transmission_techs_template:
+rule transmission_techs_and_locations_template:
     message: "Link {wildcards.resolution} direct neighbours and neighbours with sea connections with transmission techs from template."
     input:
-        template = techs_template_dir + "transmission/electricity-transmission.yaml",
+        template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja",
         units = rules.units.output[0]
     params:
         scaling_factors = config["scaling-factors"],