Magic Variables pass through (#138)

* Magic Variables pass through

* Modify DispatchManager

* Fix naming error on dispatch vars

* Fix typo

* Remove commented code

* Regold storage test

* Add documentation

src/Cases.py

@@ -19,6 +19,9 @@
 from dispatch.Factory import known as known_dispatchers
 from dispatch.Factory import get_class as get_dispatcher
 
+from ValuedParams import factory as vp_factory
+from ValuedParamHandler import ValuedParamHandler
+
 from validators.Factory import known as known_validators
 from validators.Factory import get_class as get_validator
 
@@ -266,6 +269,24 @@ def get_input_specs(cls):
     optimizer.addSub(type_sub)
     input_specs.addSub(optimizer)
 
+    # Add magic variables that will be passed to the outer and inner.
+    dispatch_vars = InputData.parameterInputFactory(
+        'dispatch_vars',
+        descr=r"This node defines a set containing additional variables"
+        "to sample that are not associated with a specific component."
+    )
+    value_param = vp_factory.make_input_specs(
+        'variable',
+        descr=r"This node defines the single additional dispatch variable used in the case."
+    )
+    value_param.addParam(
+        'name',
+        param_type=InputTypes.StringType,
+        descr=r"The unique name of the dispatch variable."
+    )
+    dispatch_vars.addSub(value_param)
+    input_specs.addSub(dispatch_vars)
+
     return input_specs
 
   def __init__(self, run_dir, **kwargs):
@@ -285,6 +306,7 @@ def __init__(self, run_dir, **kwargs):
     self.dispatcher = None      # type of dispatcher to use
     self.validator_name = None  # name of dispatch validation to use
     self.validator = None       # type of dispatch validation to use
+    self.dispatch_vars = {}     # non-component optimization ValuedParams
 
     self.outerParallel = 0     # number of outer parallel runs to use
     self.innerParallel = 0     # number of inner parallel runs to use
@@ -367,6 +389,12 @@ def read_input(self, xml):
         self.validator.read_input(vld)
       elif item.getName() == 'optimization_settings':
         self._optimization_settings = self._read_optimization_settings(item)
+      elif item.getName() == 'dispatch_vars':
+        for node in item.subparts:
+          var_name = node.parameterValues['name']
+          vp = ValuedParamHandler(var_name)
+          vp.read(var_name, node, self.get_mode())
+          self.dispatch_vars[var_name] = vp
 
     # checks
     if self._mode is None:
@@ -649,6 +677,14 @@ def get_hist_length(self):
     """
     return self._hist_len
 
+  def get_dispatch_var(self, name):
+    """
+      Accessor
+      @ In, name, str, the name of the dispatch_var
+      @ Out, dispatch_var, ValuedParamHandler, a ValuedParam object.
+    """
+    return self.dispatch_vars[name]
+
   #### API ####
   def write_workflows(self, components, sources, loc):
     """

src/DispatchManager.py

@@ -117,13 +117,8 @@ def extract_variables(self, raven, raven_dict):
     # TODO references to all ValuedParams should probably be registered somewhere
     # like maybe in the VPFactory, then we can loop through and look for info
     # that we know from Outer and fill in the blanks? Maybe?
-
-    # TODO magic keywords (e.g. verbosity, MAX_TIMES, MAX_YEARS, ONLY_DISPATCH, etc)
-    # TODO other arbitrary constants, such as sampled values from Outer needed in Inner?
-    magics = ['NPP_bid_adjust', 'HTSE_built_capacity', 'IES_delta_cap'] # XXX FIXME TODO
-    # XXX get "other opt vars" off the CASE itself, rather than hard coding.
-    for magic in magics:
-      val = getattr(raven, magic, None)
+    for magic in self._case.dispatch_vars.keys():
+      val = getattr(raven, f'{magic}_dispatch', None)
       if val is not None:
         pass_vars[magic] = float(val)
 

templates/template_driver.py

@@ -264,7 +264,19 @@ def _modify_outer_vargroups(self, template, case, components, sources):
     var_groups = template.find('VariableGroups')
     # capacities
     caps = var_groups[0]
-    caps.text = ', '.join(f'{x.name}_capacity' for x in components if (x.get_capacity(None, raw=True).type not in ['Function', 'SyntheticHistory']))
+    var_list = []
+
+    # Add component opt vars
+    for comp in components:
+      comp_cap_type = comp.get_capacity(None, raw=True).type
+      if comp_cap_type  not in ['Function', 'ARMA']:
+        var_list.append(f'{comp.name}_capacity')
+
+    # Add dispatch opt vars
+    for var in case.dispatch_vars.keys():
+      var_list.append(f'{var}_dispatch')
+    caps.text = ', '.join(var_list)
+
     # outer results
     if case._optimization_settings is not None:
       group_outer_results = var_groups.find(".//Group[@name='GRO_outer_results']")
@@ -397,25 +409,34 @@ def _modify_outer_models(self, template, case, components):
     # conversion script
     conv = raven.find('conversion').find('input')
     conv.attrib['source'] = '../write_inner.py'
+
     # aliases
     text = 'Samplers|MonteCarlo@name:mc_arma_dispatch|constant@name:{}_capacity'
     for component in components:
       name = component.name
       attribs = {'variable':'{}_capacity'.format(name), 'type':'input'}
       new = xmlUtils.newNode('alias', text=text.format(name), attrib=attribs)
       raven.append(new)
-    # if debug, grab the dispatch output instead of the summary
-    if case.debug['enabled']:
-      raven.find('outputDatabase').text = 'disp_full'
 
+    # Now we check for any non-component dispatch variables and assign aliases
+    text = 'Samplers|MonteCarlo@name:mc_arma_dispatch|constant@name:{}_dispatch'
+    for name in case.dispatch_vars.keys():
+      attribs = {'variable': f'{name}_dispatch', 'type':'input'}
+      new = xmlUtils.newNode('alias', text=text.format(name), attrib=attribs)
+      raven.append(new)
 
     # label aliases placed inside models
     text = 'Samplers|MonteCarlo@name:mc_arma_dispatch|constant@name:{}_label'
-    for var, _ in self.__case.get_labels().items():
-      attribs = {'variable': '{}_label'.format(var), 'type':'input'}
-      new = xmlUtils.newNode('alias', text=text.format(var), attrib=attribs)
+    for label in case.get_labels().keys():
+      attribs = {'variable': f'{label}_label', 'type':'input'}
+      new = xmlUtils.newNode('alias', text=text.format(label), attrib=attribs)
       raven.append(new)
 
+    # if debug, grab the dispatch output instead of the summary
+    if case.debug['enabled']:
+      raven.find('outputDatabase').text = 'disp_full'
+
+
   def _modify_outer_outstreams(self, template, case, components, sources):
     """
       Defines modifications to the OutStreams of outer.xml RAVEN input file.
@@ -496,6 +517,17 @@ def _modify_outer_samplers(self, template, case, components):
     else:
       sampler = 'opt'
 
+    for key, value in case.dispatch_vars.items():
+      var_name = self.namingTemplates['variable'].format(unit=key, feature='dispatch')
+      vals = value.get_value(debug=case.debug['enabled'])
+      if isinstance(vals, list):
+        dist, xml = self._create_new_sweep_capacity(key, var_name, vals, sampler)
+        dists_node.append(dist)
+        if case.get_mode() == 'sweep':
+          samps_node.append(xml)
+        else:
+          samps_node.append(xml)
+
     for component in components:
       interaction = component.get_interaction()
       # NOTE this algorithm does not check for everthing to be swept! Future work could expand it.
@@ -600,7 +632,10 @@ def _create_new_sweep_capacity(self, comp_name, var_name, capacities, sampler):
       @ Out, xml, xml.etree.ElementTree,Element, XML for sampled variable
     """
     # distribution
-    dist_name = self.namingTemplates['distribution'].format(unit=comp_name, feature='capacity')
+    if 'capacity' in var_name:
+      dist_name = self.namingTemplates['distribution'].format(unit=comp_name, feature='capacity')
+    else:
+      dist_name = self.namingTemplates['distribution'].format(unit=comp_name, feature='dispatch')
     dist = copy.deepcopy(self.dist_template)
     dist.attrib['name'] = dist_name
     min_cap = min(capacities)
@@ -782,6 +817,7 @@ def _modify_inner_components(self, template, case, components):
     var_groups = template.find('VariableGroups')
     for tag in ['capacities', 'init_disp', 'full_dispatch']:
       groups[tag] = var_groups.find(".//Group[@name='GRO_{}']".format(tag))
+
     # change inner input due to components requested
     for component in components:
       name = component.name

tests/integration_tests/workflows/storage/gold/Sweep_Runs_o/sweep.csv

@@ -1,3 +1,5 @@
-steamer_capacity,steam_storage_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,ProbabilityWeight,ProbabilityWeight-steamer_capacity,PointProbability,prefix
-1.0,100.0,-99.0,-2.0,-1e+200,27.3667638372,1.56343784157e-08,27.3667638291,27.3667638552,27.3667638272,27.3667638272,27.3667638552,3.0,2.44433788446e-16,0.5,0.5,0.010101010101,1
-100.0,100.0,-99.0,-2.0,-1e+200,621.494308544,3.98747544645e-07,621.494308515,621.494308957,621.494308161,621.494308161,621.494308957,3.0,1.5899960436e-13,0.5,0.5,0.010101010101,2
+steamer_capacity,steam_storage_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,NPP_bid_adjust_dispatch,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,ProbabilityWeight-NPP_bid_adjust_dispatch,prefix,PointProbability,ProbabilityWeight-steamer_capacity,ProbabilityWeight
+1.0,100.0,-99.0,-2.0,-1e+200,1.0,27.3667638372,1.56343784157e-08,27.3667638291,27.3667638552,27.3667638272,27.3667638272,27.3667638552,3.0,2.44433788446e-16,0.5,1,0.000102030405061,0.5,0.25
+100.0,100.0,-99.0,-2.0,-1e+200,1.0,621.494308544,3.98747544645e-07,621.494308515,621.494308957,621.494308161,621.494308161,621.494308957,3.0,1.5899960436e-13,0.5,2,0.000102030405061,0.5,0.25
+1.0,100.0,-99.0,-2.0,-1e+200,100.0,27.3667638372,1.56343784157e-08,27.3667638291,27.3667638552,27.3667638272,27.3667638272,27.3667638552,3.0,2.44433788446e-16,0.5,3,0.000102030405061,0.5,0.25
+100.0,100.0,-99.0,-2.0,-1e+200,100.0,621.494308544,3.98747544645e-07,621.494308515,621.494308957,621.494308161,621.494308161,621.494308957,3.0,1.5899960436e-13,0.5,4,0.000102030405061,0.5,0.25

tests/integration_tests/workflows/storage/heron_input.xml

@@ -28,6 +28,11 @@
     <dispatcher>
       <pyomo><debug_mode>True</debug_mode></pyomo>
     </dispatcher>
+    <dispatch_vars>
+      <variable name='NPP_bid_adjust'>
+        <sweep_values>1, 100</sweep_values>
+      </variable>
+    </dispatch_vars>
   </Case>
 
   <Components>