Merge pull request #70 from PaulTalbot-INL/arma_macro_len

Arma macro len

.gitignore

@@ -11,6 +11,7 @@ __pycache__
 
 # install stuff
 /.ravenconfig.xml
+*.ravenStatus
 
 # docs stuff
 *.aux

src/Cases.py

@@ -294,6 +294,10 @@ def initialize(self, components, sources):
       @ In, sources, list, HERON sources (placeholders)
       @ Out, None
     """
+    # check sources
+    for src in sources:
+      src.checkValid(self, components, sources)
+    # dispatcher
     self.dispatcher.initialize(self, components, sources)
 
   def __repr__(self):

src/DispatchManager.py

@@ -100,12 +100,9 @@ def extract_variables(self, raven, raven_dict):
     else:
       # NOTE this should ONLY BE POSSIBLE if no ARMAs are in use!
       pass
-      # FIXME index isn't always "time" ...
-      #time = getattr(raven, 'time', None)
-      #if time is not None:
-      #  pass_vars['time'] = time
 
     # variable for "time" discretization, if present
+    year_var = self._case.get_year_name()
     time_var = self._case.get_time_name()
     time_vals = getattr(raven, time_var, None)
     if time_vals is not None:
@@ -129,6 +126,16 @@ def extract_variables(self, raven, raven_dict):
         pass_vars[f'{comp.name}_capacity'] = update_capacity
     # TODO other case, component properties
 
+    # check macro parameter
+    if year_var in dir(raven):
+      year_vals = getattr(raven, year_var)
+      year_size = year_vals.size
+      project_life = hutils.get_project_lifetime(self._case, self._components) - 1 # 1 for construction year
+      if year_size != project_life:
+        raise RuntimeError(f'Provided macro variable "{year_var}" is length {year_size}, ' +
+                           f'but expected project life is {project_life}! ' +
+                           f'"{year_var}" values: {year_vals}')
+
     # load ARMA signals
     for source in self._sources:
       if source.is_type('ARMA'):
@@ -367,7 +374,6 @@ def _do_dispatch(self, meta, all_structure, project_life, interp_years, segs, se
     print('****************************************')
     return dispatch_results, cf_metrics
 
-
   def _build_econ_objects(self, heron_case, heron_components, project_life):
     """
       Generates CashFlow.CashFlow instances from HERON CashFlow instances
@@ -494,46 +500,8 @@ def _get_structure(self, raven_vars):
       # only need ARMA information, not Functions
       if not source.is_type('ARMA'):
         continue
-      structure = {}
+      structure = hutils.get_synthhist_structure(source._target_file)
       all_structure['details'][source] = structure
-      name = source.name
-      obj = pk.load(open(source._target_file, 'rb'))
-      meta = obj.writeXML().getRoot()
-
-      # interpolation
-      itp_node = meta.find('InterpolatedMultiyearROM') # FIXME isn't this multicycle sometimes?
-      if itp_node:
-        # read macro parameters
-        macro_id = itp_node.find('MacroParameterID').text.strip()
-        structure['macro'] = {'id': macro_id,
-                              'num': int(itp_node.find('MacroSteps').text),
-                              'first': int(itp_node.find('MacroFirstStep').text),
-                              'last': int(itp_node.find('MacroLastStep').text),
-                             }
-        macro_nodes = meta.findall('MacroStepROM')
-      else:
-        macro_nodes = [meta]
-
-      # clusters
-      structure['clusters'] = {}
-      for macro in macro_nodes:
-        if itp_node:
-          ma_id = int(macro.attrib[structure['macro']['id']])
-        else:
-          ma_id = 0
-        clusters_info = []
-        structure['clusters'][ma_id] = clusters_info
-        cluster_nodes = macro.findall('ClusterROM')
-        if cluster_nodes:
-          for cl_node in cluster_nodes:
-            cl_info = {'id': int(cl_node.attrib['cluster']),
-                       'represents': cl_node.find('segments_represented').text.split(','),
-                       'indices': list(int(x) for x in cl_node.find('indices').text.split(','))
-                      }
-            clusters_info.append(cl_info)
-
-      # TODO segments
-      structure['segments'] = {}
 
     # TODO check consistency between ROMs?
     # for now, just summarize what we found -> take it from the first source

src/Placeholders.py

@@ -16,7 +16,7 @@
 
 framework_path = hutils.get_raven_loc()
 sys.path.append(framework_path)
-from utils import InputData, utils, InputTypes
+from utils import InputData, InputTypes, utils, xmlUtils
 
 class Placeholder(Base):
   """
@@ -58,12 +58,37 @@ def read_input(self, xml):
     self.name = specs.parameterValues['name']
     self._source = specs.value
     # check source exists
-    ## -> check it against the input file location, not based on cwd
-    self._target_file = os.path.abspath(os.path.join(self._workingDir, self._source))
+    if self._source.startswith('%HERON%'):
+      # magic word for "relative to HERON root"
+      heron_path = hutils.get_heron_loc()
+      self._target_file = os.path.abspath(self._source.replace('%HERON%', heron_path))
+    else:
+      # check absolute path
+      rel_interp = os.path.abspath(os.path.join(self._workingDir, self._source))
+      if os.path.isfile(rel_interp):
+        self._target_file = rel_interp
+      else:
+        # check absolute path
+        abs_interp = os.path.abspath(self._source)
+        if os.path.isfile(abs_interp):
+          self._target_file = abs_interp
+    # check source
     if not os.path.isfile(self._target_file):
-      self.raiseAnError(IOError, f'File not found for <DataGenerator><{self._type}> named "{self.name}": "{self._target_file}"')
+      self.raiseAnError(IOError, f'File not found for <DataGenerator><{self._type}> named "{self.name}".' +
+                        f'\nLooked in: "{self._target_file}"' +
+                        f'\nGiven location: "{self._source}"')
     return specs
 
+  def checkValid(self, case, components, sources):
+    """
+      Check validity of placeholder given rest of system
+      @ In, case, HERON.Case, case
+      @ In, case, list(HERON.Component), components
+      @ In, sources, list(HERON.Placeholder), sources
+      @ Out, None
+    """
+    pass # overwrite to check
+
   def print_me(self, tabs=0, tab='  '):
     """
       Prints info about self
@@ -113,7 +138,11 @@ def get_input_specs(cls):
     """
     specs = InputData.parameterInputFactory('ARMA', contentType=InputTypes.StringType, ordered=False, baseNode=None,
         descr=r"""This data source is a source of synthetically-generated histories trained by RAVEN.
-              The RAVEN ARMA ROM should be trained and serialized before using it in HERON.""")
+              The RAVEN ARMA ROM should be trained and serialized before using it in HERON. The text
+              of this node indicates the location of the serialized ROM. This location is usually relative
+              with respect to the HERON XML input file; however, a full absolute path can be used,
+              or the path can be prepended with ``\%HERON\%'' to be relative to the installation
+              directory of HERON.""")
     specs.addParam('name', param_type=InputTypes.StringType, required=True,
         descr=r"""identifier for this data source in HERON and in the HERON input file. """)
     specs.addParam('variable', param_type=InputTypes.StringListType, required=True,
@@ -134,6 +163,8 @@ def __init__(self, **kwargs):
     self._type = 'ARMA'
     self._var_names = None # variables from the ARMA to use
     self.eval_mode = None # ARMA evaluation style (clustered, full, truncated)
+    self.needs_multiyear = None # if not None, then this is a 1-year ARMA that needs multiyearing
+    self.limit_interp = None # if not None, gives the years to limit this interpolated ROM to
 
   def read_input(self, xml):
     """
@@ -146,15 +177,36 @@ def read_input(self, xml):
     self.eval_mode = specs.parameterValues.get('evalMode', 'clustered')
     # check that the source ARMA exists
 
-  def interpolation(self, x, y):
+  def checkValid(self, case, components, sources):
     """
-      Passthrough to numpy interpolation
-      @ In, x, np.array, original values
-      @ In, y, float, target input value
+      Check validity of placeholder given rest of system
+      @ In, case, HERON.Case, case
+      @ In, case, list(HERON.Component), components
+      @ In, sources, list(HERON.Placeholder), sources
+      @ Out, None
     """
-
-    return interpolate.interp1d(x, y)
-
+    print(f'Checking ROM at "{self._target_file}" ...')
+    structure = hutils.get_synthhist_structure(self._target_file)
+    interpolated = 'macro' in structure
+    clustered = bool(structure['clusters'])
+    # segmented = bool(structure['segments']) # TODO
+    print(f'For DataGenerator <{self._type}> "{self.name}", detected: ' +
+          f'{"" if interpolated else "NOT"} interpolated, ' +
+          f'{"" if clustered else "NOT"} clustered.')
+    # expect that project life == num macro years
+    project_life = hutils.get_project_lifetime(case, components) - 1 # one less for construction year
+    if interpolated:
+      # if interpolated, needs more checking
+      interp_years = structure['macro']['num']
+      if interp_years >= project_life:
+        print(f' -> "{self.name}" interpolates {interp_years} macro steps, and project life is {project_life}, so histories will be trunctated.')
+        self.limit_interp = project_life
+      else:
+        raise RuntimeError(f' -> "{self.name}" interpolates {interp_years} macro steps, but project life is {project_life}!')
+    else:
+      # if single year, we can use multiyear so np
+      print(f' -> "{self.name}" will be extended to project life ({project_life}) macro steps using <Multicycle>.')
+      self.needs_multiyear = project_life
 
 
 
@@ -174,7 +226,11 @@ def get_input_specs(cls):
     specs = InputData.parameterInputFactory('Function', contentType=InputTypes.StringType,
         ordered=False, baseNode=None,
         descr=r"""This data source is a custom Python function to provide derived values.
-              Python functions have access to the variables within the dispatcher.""")
+              Python functions have access to the variables within the dispatcher. The text
+              of this node indicates the location of the python file. This location is usually relative
+              with respect to the HERON XML input file; however, a full absolute path can be used,
+              or the path can be prepended with ``\%HERON\%'' to be relative to the installation
+              directory of HERON.""")
     specs.addParam('name', param_type=InputTypes.StringType, required=True,
         descr=r"""identifier for this data source in HERON and in the HERON input file. """)
     return specs

src/_utils.py

@@ -9,6 +9,14 @@
 import importlib
 import xml.etree.ElementTree as ET
 
+def get_heron_loc():
+  """
+    Return HERON location
+    @ In, None
+    @ Out, loc, string, absolute location of HERON
+  """
+  return os.path.abspath(os.path.join(__file__, '..', '..'))
+
 def get_raven_loc():
   """
     Return RAVEN location
@@ -72,6 +80,56 @@ def get_project_lifetime(case, components):
   econ_settings.setParams(econ_params)
   return getProjectLength(econ_settings, econ_comps)
 
+def get_synthhist_structure(fpath):
+  """
+    Extracts synthetic history info from ROM (currently ARMA ROM)
+    @ In, fpath, str, full absolute path to serialized ROM
+    @ Out, structure, dict, derived structure from reading ROM XML
+  """
+  # TODO could this be a function of the ROM itself?
+  # TODO or could we interrogate the ROM directly instead of the XML?
+  raven_loc = get_raven_loc()
+  scripts_path = os.path.join(raven_loc, '..', 'scripts')
+  sys.path.append(scripts_path)
+  from externalROMloader import ravenROMexternal as ravenROM
+  rom = ravenROM(fpath, raven_loc).rom
+  meta = rom.writeXML().getRoot()
+  structure = {}
+  # interpolation information
+  interp_node = meta.find('InterpolatedMultiyearROM')
+  if interp_node:
+    macro_id = interp_node.find('MacroParameterID').text.strip()
+    structure['macro'] = {'id': macro_id,
+                          'num': int(interp_node.find('MacroSteps').text),
+                          'first': int(interp_node.find('MacroFirstStep').text),
+                          'last': int(interp_node.find('MacroLastStep').text),
+                          }
+    macro_nodes = meta.findall('MacroStepROM')
+  else:
+    macro_nodes = [meta]
+  # cluster information
+  structure['clusters'] = {}
+  for macro in macro_nodes:
+    if interp_node:
+      macro_index = int(macro.attrib[macro_id])
+    else:
+      macro_index = 0
+    clusters_info = [] # data dict for each macro step
+    structure['clusters'][macro_index] = clusters_info
+    cluster_nodes = macro.findall('ClusterROM')
+    if cluster_nodes:
+      for node in cluster_nodes:
+        info = {'id': int(node.attrib['cluster']),
+                'represents': node.find('segments_represented').text.split(','),
+                'indices': list(int(x) for x in node.find('indices').text.split(','))
+               }
+        clusters_info.append(info)
+  # segment information
+  # -> TODO
+  structure['segments'] = {}
+  return structure
+
+
 if __name__ == '__main__':
   try:
     action = sys.argv[1]

templates/template_driver.py

@@ -773,20 +773,18 @@ def _iostep_load_rom(self, template, case, components, source):
     self._updateCommaSeperatedList(template.find('RunInfo').find('Sequence'), new_step.attrib['name'], position=0)
     # add the model
     model = xmlUtils.newNode('ROM', attrib={'name':rom_name, 'subType':'pickledROM'})
-    ## update the ARMA model to sample a number of years equal to the ProjectLife from CashFlow
-    #comp_lifes = list(comp.get_economics().get_lifetime() for comp in components)
-    #req_proj_life = case.get_econ().get('ProjectLife', None)
     econ_comps = list(comp.get_economics() for comp in components)
     econ_global_params = case.get_econ(econ_comps)
     econ_global_settings = CashFlows.GlobalSettings()
     econ_global_settings.setParams(econ_global_params)
-    #project_life = getProjectLength(econ_global_settings, econ_comps) - 1 # skip construction year
-    #multiyear = xmlUtils.newNode('Multiyear')
-    #multiyear.append(xmlUtils.newNode('years', text=project_life))
-    # TODO FIXME XXX growth param ????
-    #model.append(multiyear)
-    ## update the ARMA model to use clustered eval mode
-    # FIXME this isn't always desired; what if it isn't clustered?
+    ## update the ARMA model to sample a number of years equal to the ProjectLife from CashFlow
+    if source.needs_multiyear is not None:
+      multiyear = xmlUtils.newNode('Multicycle')
+      multiyear.append(xmlUtils.newNode('cycles', text=source.needs_multiyear))
+      model.append(multiyear)
+    if source.limit_interp is not None:
+      model.append(xmlUtils.newNode('maxCycles', text=source.limit_interp))
+    # change eval mode?
     if source.eval_mode == 'clustered':
       model.append(xmlUtils.newNode('clusterEvalMode', text='clustered'))
     template.find('Models').append(model)

tests/integration_tests/ARMA/Sine/Data.csv

@@ -1,2 +1,3 @@
 scaling,Year,filename
 1,10,Data_0.csv
+1,12,Data_0.csv

tests/integration_tests/ARMA/Sine_Hour/Data.csv

@@ -1,2 +1,3 @@
 scaling,YEAR,filename
 1,2015,Data_0.csv
+1,2018,Data_0.csv

tests/integration_tests/cashflows/heron_input.xml → tests/integration_tests/mechanics/cashflows/heron_input.xml

@@ -102,7 +102,7 @@
     </Components>
 
     <DataGenerators>
-      <ARMA name='flex' variable="Signal">../ARMA/Sine/arma.pk</ARMA>
+      <ARMA name='flex' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma_long.pk</ARMA>
       <Function name="functions">functions.py</Function>
     </DataGenerators>
 </HERON>

tests/integration_tests/custom_dispatcher/heron_input.xml → tests/integration_tests/mechanics/custom_dispatcher/heron_input.xml

@@ -111,7 +111,7 @@
   </Components>
 
   <DataGenerators>
-    <ARMA name='Price' variable="Signal">../ARMA/Sine/arma.pk</ARMA>
+    <ARMA name='Price' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma.pk</ARMA>
     <Function name="transfers">transfers.py</Function>
   </DataGenerators>
 

tests/integration_tests/labels/heron_input.xml → tests/integration_tests/mechanics/labels/heron_input.xml

@@ -122,7 +122,7 @@
   </Components>
 
   <DataGenerators>
-    <ARMA name='Price' variable="Signal">../ARMA/Sine_Hour/arma.pk</ARMA>
+    <ARMA name='Price' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine_Hour/arma.pk</ARMA>
     <Function name="transfers">transfers.py</Function>
   </DataGenerators>
 

tests/integration_tests/min_demand/heron_input.xml → tests/integration_tests/mechanics/min_demand/heron_input.xml

@@ -103,7 +103,7 @@
     </Components>
 
     <DataGenerators>
-      <ARMA name='flex' variable="Signal">../ARMA/Sine/arma.pk</ARMA>
+      <ARMA name='flex' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma.pk</ARMA>
       <Function name="transfers">transfers.py</Function>
     </DataGenerators>
 </HERON>

tests/integration_tests/multimarket_fix_price/heron_input.xml → tests/integration_tests/mechanics/multimarket_fix_price/heron_input.xml

@@ -101,7 +101,7 @@
   </Components>
 
   <DataGenerators>
-    <ARMA name='Speed' variable="Signal">../ARMA/Sine/arma.pk</ARMA>
+    <ARMA name='Speed' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma.pk</ARMA>
     <Function name="transfers">transfers.py</Function> <!-- TODO specify the variables needed? -->
   </DataGenerators>
 

tests/integration_tests/production_steady/heron_input.xml → tests/integration_tests/mechanics/production_steady/heron_input.xml

@@ -134,7 +134,7 @@
   </Components>
 
   <DataGenerators>
-    <ARMA name='Speed' variable="Signal">../ARMA/Sine/arma.pk</ARMA>
+    <ARMA name='Speed' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma.pk</ARMA>
     <Function name="transfers">transfers.py</Function> <!-- TODO specify the variables needed? -->
   </DataGenerators>