diff --git a/CHANGELOG.md b/CHANGELOG.md index d14fb5b1..bbb4b0d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ * **UPDATED** structure of YAML templates and parametrisation: * Parametrisation moved to eurocalliopelib. * Rules to parametrise split into smaller technology-specific rules, to ensure inputs are directly relevant to the files being parametrised. - * YAML templates restructured to match structure of final model (see `Updated (models) above`); + * YAML templates restructured to match structure of final model (see `Updated (models) above`) and given `.jinja` suffix to allow for IDE syntax highlighting (#404); * **UPDATE** cluster sync infrastructure to retain file permission defaults on the cluster. This change improves team collaboration, as default group settings will apply to the files on the cluster (#214). * **UPDATE** the declaration of required cluster resources. Moving away from a mechanism that is deprecated in Snakemake (#211). * **UPDATE** default Snakemake profile to be activated automatically, for convenience (#264, #268). diff --git a/Snakefile b/Snakefile index c8f671aa..96c93521 100644 --- a/Snakefile +++ b/Snakefile @@ -30,11 +30,9 @@ include: "./rules/modules.smk" min_version("8.10") localrules: all, clean wildcard_constraints: - resolution = "continental|national|regional|ehighways" - -ruleorder: area_to_capacity_limits > hydro_capacities > biofuels > nuclear_regional_capacity > dummy_tech_locations_template -ruleorder: bio_techs_and_locations_template > techs_and_locations_template -ruleorder: create_controlled_road_transport_annual_demand_and_installed_capacities > dummy_tech_locations_template + resolution = "continental|national|regional|ehighways", + group_and_tech = "(demand|storage|supply|transmission)\/\w+" +ruleorder: module_with_location_specific_data > module_without_location_specific_data ALL_CF_TECHNOLOGIES = [ "wind-onshore", "wind-offshore", "open-field-pv", @@ -96,32 +94,30 @@ rule all_tests: ) -rule dummy_tech_locations_template: # needed to provide `techs_and_locations_template` with a locational CSV linked to each technology that has no location-specific data to define. - message: "Create empty {wildcards.resolution} location-specific data file for the {wildcards.tech_group} tech `{wildcards.tech}`." # Update ruleorder at the top of the file if you instead want the techs_and_locations_template rule to be used to generate a file - input: rules.locations_template.output.csv - output: "build/data/{resolution}/{tech_group}/{tech}.csv" - conda: "envs/shell.yaml" - shell: "cp {input} {output}" - - -rule techs_and_locations_template: - message: "Create {wildcards.resolution} definition file for the {wildcards.tech_group} tech `{wildcards.tech}`." +rule module_with_location_specific_data: + message: "Create {wildcards.resolution} definition file for {wildcards.group_and_tech}." input: - template = techs_template_dir + "{tech_group}/{tech}.yaml", - locations = "build/data/{resolution}/{tech_group}/{tech}.csv" + template = techs_template_dir + "{group_and_tech}.yaml.jinja", + locations = "build/data/{resolution}/{group_and_tech}.csv" params: scaling_factors = config["scaling-factors"], capacity_factors = config["capacity-factors"]["average"], max_power_densities = config["parameters"]["maximum-installable-power-density"], heat_pump_shares = config["parameters"]["heat-pump"]["heat-pump-shares"], wildcard_constraints: - tech_group = "(?!transmission).*" # i.e. all but transmission + # Exclude all outputs that have their own `techs_and_locations_template` implementation + group_and_tech = "(?!transmission\/|supply\/biofuel).*" conda: "envs/default.yaml" - output: "build/models/{resolution}/techs/{tech_group}/{tech}.yaml" + output: "build/models/{resolution}/techs/{group_and_tech}.yaml" script: "scripts/template_techs.py" +use rule module_with_location_specific_data as module_without_location_specific_data with: + # For all cases where we don't have any location-specific data that we want to supply to the template + input: + template = techs_template_dir + "{group_and_tech}.yaml.jinja", + locations = rules.locations_module.output.csv -rule no_params_model_template: +rule module_without_specific_data: message: "Create {wildcards.resolution} configuration files from templates where no parameterisation is required." input: template = model_template_dir + "{template}", @@ -132,8 +128,8 @@ rule no_params_model_template: shell: "cp {input.template} {output}" -rule no_params_template: - message: "Create non-model files from templates where no parameterisation is required." +rule auxiliary_files: + message: "Create auxiliary output files (i.e. those not used to define a Calliope model) from templates where no parameterisation is required." input: template = template_dir + "{template}", output: "build/models/{template}" @@ -143,16 +139,16 @@ rule no_params_template: shell: "cp {input.template} {output}" -rule model_template: +rule model: message: "Generate top-level {wildcards.resolution} model configuration file from template" input: - template = model_template_dir + "example-model.yaml", - non_model_files = expand( + template = model_template_dir + "example-model.yaml.jinja", + auxiliary_files = expand( "build/models/{template}", template=["environment.yaml", "README.md"] ), - input_files = expand( - "build/models/{{resolution}}/{input_file}", - input_file=[ + modules = expand( + "build/models/{{resolution}}/{module}", + module=[ "interest-rate.yaml", "locations.yaml", "techs/demand/electricity.yaml", @@ -187,9 +183,9 @@ rule model_template: "build/models/{resolution}/timeseries/demand/demand-shape-equals-ev.csv", "build/models/{resolution}/timeseries/demand/plugin-profiles-ev.csv", ), - optional_input_files = lambda wildcards: expand( - f"build/models/{wildcards.resolution}/{{input_file}}", - input_file=[ + optional_modules = lambda wildcards: expand( + f"build/models/{wildcards.resolution}/{{module}}", + module=[ "techs/transmission/electricity-linked-neighbours.yaml", ] + ["techs/transmission/electricity-entsoe.yaml" for i in [None] if wildcards.resolution == "national"] ) diff --git a/rules/biofuels.smk b/rules/biofuels.smk index a93940a1..7dcc89c2 100644 --- a/rules/biofuels.smk +++ b/rules/biofuels.smk @@ -1,6 +1,5 @@ """Rules related to biofuels.""" - rule download_biofuel_potentials_and_costs: message: "Download raw biofuel potential and cost data." params: url = config["data-sources"]["biofuel-potentials-and-costs"] @@ -52,10 +51,10 @@ rule biofuels: script: "../scripts/biofuels/allocate.py" -rule bio_techs_and_locations_template: +rule biofuel_tech_module: message: "Create biofuel tech definition file from template." input: - template = techs_template_dir + "supply/biofuel.yaml", + template = techs_template_dir + "supply/biofuel.yaml.jinja", biofuel_cost = "build/data/regional/biofuel/{scenario}/costs-eur-per-mwh.csv".format( scenario=config["parameters"]["jrc-biofuel"]["scenario"] ), diff --git a/rules/shapes.smk b/rules/shapes.smk index 0632ffe5..47404b6b 100644 --- a/rules/shapes.smk +++ b/rules/shapes.smk @@ -140,10 +140,10 @@ rule eez: """ -rule locations_template: +rule locations_module: message: "Generate locations configuration file for {wildcards.resolution} resolution from template." input: - template = model_template_dir + "locations.yaml", + template = model_template_dir + "locations.yaml.jinja", shapes = rules.units.output[0] output: yaml = "build/models/{resolution}/locations.yaml", diff --git a/rules/transmission.smk b/rules/transmission.smk index 16f489b8..633eaf38 100644 --- a/rules/transmission.smk +++ b/rules/transmission.smk @@ -20,11 +20,11 @@ rule entsoe_tyndp_xlsx: shell: "unzip -o {input} 'TYNDP-2020-Scenario-Datafile.xlsx' -d build/data/national" -rule transmission_entsoe_tyndp_template: +rule transmission_entsoe_tyndp_tech_module: message: "Create YAML file of national-scale links with ENTSO-E TYNDP net-transfer capacities" input: - template = techs_template_dir + "transmission/electricity-transmission.yaml", - locations = rules.locations_template.output.csv, + template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja", + locations = rules.locations_module.output.csv, entsoe_tyndp = rules.entsoe_tyndp_xlsx.output[0] params: scenario = config["parameters"]["entsoe-tyndp"]["scenario"], @@ -39,10 +39,10 @@ rule transmission_entsoe_tyndp_template: script: "../scripts/transmission/template_transmission_entsoe_tyndp.py" -rule link_locations_with_transmission_techs_template: +rule transmission_linked_neighbours_tech_module: message: "Link {wildcards.resolution} direct neighbours and neighbours with sea connections with transmission techs from template." input: - template = techs_template_dir + "transmission/electricity-transmission.yaml", + template = techs_template_dir + "transmission/electricity-transmission.yaml.jinja", units = rules.units.output[0] params: scaling_factors = config["scaling-factors"], diff --git a/scripts/template_model.py b/scripts/template_model.py index dca1c727..c5f3e738 100644 --- a/scripts/template_model.py +++ b/scripts/template_model.py @@ -5,8 +5,8 @@ from eurocalliopelib.template import parametrise_template -def construct_model(path_to_template, path_to_output, input_files, resolution, year): - input_files = sorted([update_path(file, resolution) for file in input_files]) +def construct_model(path_to_template, path_to_output, modules, resolution, year): + input_files = sorted([update_path(file, resolution) for file in modules]) return parametrise_template( path_to_template, @@ -26,7 +26,7 @@ def update_path(path_string, resolution): if __name__ == "__main__": construct_model( path_to_template=snakemake.input.template, - input_files=snakemake.input.input_files, + modules=snakemake.input.modules, resolution=snakemake.wildcards.resolution, year=snakemake.params.year, path_to_output=snakemake.output[0], diff --git a/templates/models/example-model.yaml b/templates/models/example-model.yaml.jinja similarity index 100% rename from templates/models/example-model.yaml rename to templates/models/example-model.yaml.jinja diff --git a/templates/models/locations.yaml b/templates/models/locations.yaml.jinja similarity index 100% rename from templates/models/locations.yaml rename to templates/models/locations.yaml.jinja diff --git a/templates/models/techs/demand/electricity.yaml b/templates/models/techs/demand/electricity.yaml.jinja similarity index 100% rename from templates/models/techs/demand/electricity.yaml rename to templates/models/techs/demand/electricity.yaml.jinja diff --git a/templates/models/techs/demand/electrified-heat.yaml b/templates/models/techs/demand/electrified-heat.yaml.jinja similarity index 100% rename from templates/models/techs/demand/electrified-heat.yaml rename to templates/models/techs/demand/electrified-heat.yaml.jinja diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml.jinja similarity index 100% rename from templates/models/techs/demand/electrified-transport.yaml rename to templates/models/techs/demand/electrified-transport.yaml.jinja diff --git a/templates/models/techs/storage/electricity.yaml b/templates/models/techs/storage/electricity.yaml.jinja similarity index 100% rename from templates/models/techs/storage/electricity.yaml rename to templates/models/techs/storage/electricity.yaml.jinja diff --git a/templates/models/techs/storage/heat.yaml b/templates/models/techs/storage/heat.yaml.jinja similarity index 100% rename from templates/models/techs/storage/heat.yaml rename to templates/models/techs/storage/heat.yaml.jinja diff --git a/templates/models/techs/storage/hydro.yaml b/templates/models/techs/storage/hydro.yaml.jinja similarity index 100% rename from templates/models/techs/storage/hydro.yaml rename to templates/models/techs/storage/hydro.yaml.jinja diff --git a/templates/models/techs/supply/biofuel.yaml b/templates/models/techs/supply/biofuel.yaml.jinja similarity index 100% rename from templates/models/techs/supply/biofuel.yaml rename to templates/models/techs/supply/biofuel.yaml.jinja diff --git a/templates/models/techs/supply/heat-from-electricity.yaml b/templates/models/techs/supply/heat-from-electricity.yaml.jinja similarity index 100% rename from templates/models/techs/supply/heat-from-electricity.yaml rename to templates/models/techs/supply/heat-from-electricity.yaml.jinja diff --git a/templates/models/techs/supply/hydro.yaml b/templates/models/techs/supply/hydro.yaml.jinja similarity index 100% rename from templates/models/techs/supply/hydro.yaml rename to templates/models/techs/supply/hydro.yaml.jinja diff --git a/templates/models/techs/supply/load-shedding.yaml b/templates/models/techs/supply/load-shedding.yaml.jinja similarity index 100% rename from templates/models/techs/supply/load-shedding.yaml rename to templates/models/techs/supply/load-shedding.yaml.jinja diff --git a/templates/models/techs/supply/nuclear.yaml b/templates/models/techs/supply/nuclear.yaml.jinja similarity index 100% rename from templates/models/techs/supply/nuclear.yaml rename to templates/models/techs/supply/nuclear.yaml.jinja diff --git a/templates/models/techs/supply/open-field-solar-and-wind-onshore.yaml b/templates/models/techs/supply/open-field-solar-and-wind-onshore.yaml.jinja similarity index 100% rename from templates/models/techs/supply/open-field-solar-and-wind-onshore.yaml rename to templates/models/techs/supply/open-field-solar-and-wind-onshore.yaml.jinja diff --git a/templates/models/techs/supply/rooftop-solar.yaml b/templates/models/techs/supply/rooftop-solar.yaml.jinja similarity index 100% rename from templates/models/techs/supply/rooftop-solar.yaml rename to templates/models/techs/supply/rooftop-solar.yaml.jinja diff --git a/templates/models/techs/supply/wind-offshore.yaml b/templates/models/techs/supply/wind-offshore.yaml.jinja similarity index 100% rename from templates/models/techs/supply/wind-offshore.yaml rename to templates/models/techs/supply/wind-offshore.yaml.jinja diff --git a/templates/models/techs/transmission/electricity-transmission.yaml b/templates/models/techs/transmission/electricity-transmission.yaml.jinja similarity index 100% rename from templates/models/techs/transmission/electricity-transmission.yaml rename to templates/models/techs/transmission/electricity-transmission.yaml.jinja