dvc.yaml

stages:
  parse_nrel118_buses:
    desc: "Parse raw bus data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_buses.py
        data/raw/nrel118/additional-files-mti-118/Buses.csv
        data/parsed/nrel118_buses.csv
    deps:
      - data/raw/nrel118/additional-files-mti-118/Buses.csv
      - src/data/parse/nrel118_buses.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/parsed/nrel118_buses.csv

  parse_nrel118_lines:
    desc: "Parse raw line data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_lines.py
        data/raw/nrel118/additional-files-mti-118/Lines.csv
        data/parsed/nrel118_lines.csv
    deps:
      - data/raw/nrel118/additional-files-mti-118/Lines.csv
      - src/data/parse/nrel118_lines.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/parsed/nrel118_lines.csv

  parse_jeas118_buses:
    desc: "Parse raw bus data from the JEAS-118 dataset"
    cmd:
      - python src/data/parse/jeas118_buses.py
        data/raw/jeas118/JEAS_IEEE118.docx
        data/parsed/jeas118_buses.csv
    deps:
      - data/raw/jeas118/JEAS_IEEE118.docx
      - src/data/parse/jeas118_buses.py
      - src/utils/converters.py
    outs:
      - data/parsed/jeas118_buses.csv

  parse_jeas118_lines:
    desc: "Parse raw line data from the JEAS-118 dataset"
    cmd:
      - python src/data/parse/jeas118_lines.py
        data/raw/jeas118/JEAS_IEEE118.docx
        data/parsed/jeas118_lines.csv
    deps:
      - data/raw/jeas118/JEAS_IEEE118.docx
      - src/data/parse/jeas118_lines.py
      - src/utils/converters.py
    outs:
      - data/parsed/jeas118_lines.csv

  parse_jeas118_trafos:
    desc: "Parse raw transformer data from the JEAS-118 dataset"
    cmd:
      - python src/data/parse/jeas118_trafos.py
        data/raw/jeas118/JEAS_IEEE118.docx
        data/parsed/jeas118_trafos.csv
    deps:
      - data/raw/jeas118/JEAS_IEEE118.docx
      - src/data/parse/jeas118_trafos.py
      - src/utils/converters.py
    outs:
      - data/parsed/jeas118_trafos.csv

  parse_jeas118_loads:
    desc: "Parse raw load data from the JEAS-118 dataset"
    cmd:
      - python src/data/parse/jeas118_loads.py
        data/raw/jeas118/JEAS_IEEE118.docx
        data/parsed/jeas118_loads.csv
    deps:
      - data/raw/jeas118/JEAS_IEEE118.docx
      - src/data/parse/jeas118_loads.py
      - src/utils/converters.py
    outs:
      - data/parsed/jeas118_loads.csv

  parse_nrel118_loads_ts:
    desc: "Parse raw time-series load data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_loads_ts.py
        "data/raw/nrel118/Input files/RT/Load"
        data/parsed/nrel118_loads_ts.csv
    deps:
      - data/raw/nrel118/Input files/RT/Load
      - src/data/parse/nrel118_loads_ts.py
      - src/utils/data_loaders/load_ts_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_loads_ts.csv

  parse_nrel118_gens:
    desc: "Parse raw generation data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_gens.py
        data/raw/nrel118/additional-files-mti-118/Generators.csv
        data/parsed/nrel118_gens.csv
    deps:
      - data/raw/nrel118/additional-files-mti-118/Generators.csv
      - src/data/parse/nrel118_gens.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - GEN_TYPES
    outs:
      - data/parsed/nrel118_gens.csv

  parse_nrel118_escalators_ts:
    desc: "Parse raw escalator time-series data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_escalators_ts.py
        data/raw/nrel118/additional-files-mti-118/Escalators.csv
        data/parsed/nrel118_escalators_ts.csv
    deps:
      - data/raw/nrel118/additional-files-mti-118/Escalators.csv
      - src/data/parse/nrel118_escalators_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_escalators_ts.csv

  parse_nrel118_hydros_nondisp_ts:
    desc: "Parse raw non-dispatchable hydro plants data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_hydros_nondisp_ts.py
        data/raw/nrel118/additional-files-mti-118/Hydro_nondipatchable.csv
        data/parsed/nrel118_hydros_nondisp_ts.csv
    deps:
      - data/raw/nrel118/additional-files-mti-118/Hydro_nondipatchable.csv
      - src/data/parse/nrel118_hydros_nondisp_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_hydros_nondisp_ts.csv

  parse_nrel118_hydros_ts:
    desc: "Parse raw time-series hydro gens data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_hydros_ts.py
        "data/raw/nrel118/Input files/Hydro"
        data/parsed/nrel118_hydros_ts.csv
    deps:
      - data/raw/nrel118/Input files/Hydro
      - src/data/parse/nrel118_hydros_ts.py
      - src/utils/data_loaders/load_ts_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_hydros_ts.csv

  parse_nrel118_solars_ts:
    desc: "Parse raw time-series solar gens data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_solars_ts.py
        "data/raw/nrel118/Input files/RT/Solar"
        data/parsed/nrel118_solars_ts.csv
    deps:
      - data/raw/nrel118/Input files/RT/Solar
      - src/data/parse/nrel118_solars_ts.py
      - src/utils/data_loaders/load_ts_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_solars_ts.csv

  parse_nrel118_winds_ts:
    desc: "Parse raw time-series wind gens data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_winds_ts.py
        "data/raw/nrel118/Input files/RT/Wind"
        data/parsed/nrel118_winds_ts.csv
    deps:
      - data/raw/nrel118/Input files/RT/Wind
      - src/data/parse/nrel118_winds_ts.py
      - src/utils/data_loaders/load_ts_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_winds_ts.csv

  parse_nrel118_outages_ts:
    desc: "Parse raw time-series outage data from the NREL-118 dataset"
    cmd:
      - python src/data/parse/nrel118_outages_ts.py
        "data/raw/nrel118/Input files/Others/GenOut.csv"
        data/parsed/nrel118_outages_ts.csv
    deps:
      - data/raw/nrel118/Input files/Others/GenOut.csv
      - src/data/parse/nrel118_outages_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - GEN_TYPES
          - DATE_FORMAT
    outs:
      - data/parsed/nrel118_outages_ts.csv

  transform_loads:
    desc: "Combine necessary load variables for the preparation stage"
    cmd:
      - python src/data/transform/loads.py
        data/parsed/nrel118_buses.csv
        data/parsed/jeas118_loads.csv
        data/transformed/loads.csv
    deps:
      - data/parsed/nrel118_buses.csv
      - data/parsed/jeas118_loads.csv
      - src/data/transform/loads.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/transformed/loads.csv

  transform_gens:
    desc: "Combine necessary gen variables for preparation stages"
    cmd:
      - python src/data/transform/gens.py
        data/parsed/nrel118_gens.csv
        data/prepared/buses.csv
        data/transformed/gens.csv
    deps:
      - data/parsed/nrel118_gens.csv
      - data/prepared/buses.csv
      - src/data/transform/gens.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/transformed/gens.csv

  transform_outages_ts:
    desc: "Modify dataset with outages"
    cmd:
      - python src/data/transform/outages_ts.py
        data/parsed/nrel118_outages_ts.csv
        data/transformed/outages_ts.csv
    deps:
      - data/parsed/nrel118_outages_ts.csv
      - src/data/transform/outages_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
    outs:
      - data/transformed/outages_ts.csv

  transform_gens_escalated_ts:
    desc: "Combine gen output limits and escalator values"
    cmd:
      - python src/data/transform/gens_escalated_ts.py
        data/transformed/gens.csv
        data/parsed/nrel118_escalators_ts.csv
        data/transformed/gens_escalated_ts.csv
    deps:
      - data/transformed/gens.csv
      - data/parsed/nrel118_escalators_ts.csv
      - src/data/transform/gens_escalated_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
    outs:
      - data/transformed/gens_escalated_ts.csv

  transform_gens_ts:
    desc: "Concat generation time-series data"
    cmd:
      - python src/data/transform/gens_ts.py
        data/transformed/gens.csv
        data/parsed/nrel118_winds_ts.csv
        data/parsed/nrel118_solars_ts.csv
        data/parsed/nrel118_hydros_ts.csv
        data/parsed/nrel118_hydros_nondisp_ts.csv
        data/transformed/gens_ts.csv
    deps:
      - data/transformed/gens.csv
      - data/parsed/nrel118_winds_ts.csv
      - data/parsed/nrel118_solars_ts.csv
      - data/parsed/nrel118_hydros_ts.csv
      - data/parsed/nrel118_hydros_nondisp_ts.csv
      - src/data/transform/gens_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
    outs:
      - data/transformed/gens_ts.csv

  prepare_buses:
    desc: "Build final bus dataset"
    cmd:
      - python src/data/prepare/buses.py
        data/parsed/nrel118_buses.csv
        data/parsed/jeas118_buses.csv
        data/raw/manual/bus_coordinates.csv
        data/prepared/buses.csv
      - python src/data/check/buses.py
        data/prepared/buses.csv
    deps:
      - data/parsed/nrel118_buses.csv
      - data/parsed/jeas118_buses.csv
      - data/raw/manual/bus_coordinates.csv
      - src/data/prepare/buses.py
      - src/data/check/buses.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/prepared/buses.csv

  prepare_branches:
    desc: "Build final dataset with line and transformer info"
    cmd:
      - python src/data/prepare/branches.py
        data/parsed/nrel118_lines.csv
        data/parsed/jeas118_lines.csv
        data/parsed/jeas118_trafos.csv
        data/prepared/buses.csv
        data/prepared/branches.csv
      - python src/data/check/branches.py
        data/prepared/branches.csv
        data/prepared/buses.csv
    deps:
      - data/parsed/nrel118_lines.csv
      - data/parsed/jeas118_lines.csv
      - data/parsed/jeas118_trafos.csv
      - data/prepared/buses.csv
      - src/data/prepare/branches.py
      - src/data/check/branches.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - S_BASE_MVA
    outs:
      - data/prepared/branches.csv

  prepare_loads:
    desc: "Build final dataset with load info"
    cmd:
      - python src/data/prepare/loads.py
        data/transformed/loads.csv
        data/prepared/loads.csv
      - python src/data/check/loads.py
        data/prepared/loads.csv
        data/prepared/buses.csv
    deps:
      - data/transformed/loads.csv
      - data/prepared/buses.csv
      - src/data/prepare/loads.py
      - src/data/check/loads.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/prepared/loads.csv

  prepare_loads_ts:
    desc: "Build final dataset with load time-series data"
    cmd:
      - python src/data/prepare/loads_ts.py
        data/transformed/loads.csv
        data/parsed/nrel118_loads_ts.csv
        data/prepared/loads_ts.csv
      - python src/data/check/loads_ts.py
        data/prepared/loads_ts.csv
        data/prepared/loads.csv
    deps:
      - data/prepared/loads.csv
      - data/transformed/loads.csv
      - data/parsed/nrel118_loads_ts.csv
      - src/data/prepare/loads_ts.py
      - src/data/check/loads_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
          - DATE_RANGE
          - FILL_METHOD
    outs:
      - data/prepared/loads_ts.csv

  prepare_gens:
    desc: "Build final dataset with generation info"
    cmd:
      - python src/data/prepare/gens.py
        data/transformed/gens.csv
        data/prepared/gens_ts.csv
        data/prepared/gens.csv
      - python src/data/check/gens.py
        data/prepared/gens.csv
        data/prepared/buses.csv
      - python src/data/check/gens_ts.py
        data/prepared/gens_ts.csv
        data/prepared/gens.csv
    deps:
      - data/prepared/buses.csv
      - data/prepared/gens_ts.csv
      - data/transformed/gens.csv
      - src/data/prepare/gens.py
      - src/data/check/gens.py
      - src/data/check/gens_ts.py
      - src/utils/data_loaders/load_df_data.py
    outs:
      - data/prepared/gens.csv

  prepare_gens_ts:
    desc: "Build final dataset with generation time-series data"
    cmd:
      - python src/data/prepare/gens_ts.py
        data/transformed/gens.csv
        data/transformed/gens_ts.csv
        data/transformed/gens_escalated_ts.csv
        data/transformed/outages_ts.csv
        data/prepared/gens_ts.csv
    deps:
      - data/transformed/gens.csv
      - data/transformed/gens_ts.csv
      - data/transformed/gens_escalated_ts.csv
      - data/transformed/outages_ts.csv
      - src/data/prepare/gens_ts.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - DATE_FORMAT
          - DATE_RANGE
          - FILL_METHOD
    outs:
      - data/prepared/gens_ts.csv

  build:
    desc: "Start building power flow cases"
    cmd:
      - python src/power_flow/building.py
        data/prepared/buses.csv
        data/prepared/branches.csv
        data/prepared/loads.csv
        data/prepared/loads_ts.csv
        data/prepared/gens.csv
        data/prepared/gens_ts.csv
        samples
    deps:
      - data/prepared/buses.csv
      - data/prepared/branches.csv
      - data/prepared/loads.csv
      - data/prepared/loads_ts.csv
      - data/prepared/gens.csv
      - data/prepared/gens_ts.csv
      - src/power_flow/building.py
      - src/power_flow/builders/base.py
      - src/power_flow/builders/pandapower.py
      - src/utils/data_loaders/load_df_data.py
    params:
      - definitions.py:
          - S_BASE_MVA
          - F_HZ
          - POWER_FLOW_ENGINE
          - DATE_FORMAT
          - SAMPLE_NAME_FORMAT
    outs:
      - samples