Add capability to handle interdependent functions in samplers/optimiz…

…ers (#2319)

* moved samplers-optimizers function tests in their own folder for clarity

* renamed

* modified test names and revisions

* moved subfolders

* Closes #2302

* usage base class function evaluation method in ensembleForward

* added error msg for detection of loop in function system

* added test for ensemble model passing strings (restart file paths) around

* removed trailing spaces

* Update ravenframework/Samplers/EnsembleForward.py

* Update ravenframework/Samplers/Sampler.py

* Update ravenframework/Samplers/Sampler.py

* Update ravenframework/Samplers/Sampler.py

* fixed comment

* we always check for isolated functions

* updated model.tex

* changed order to reflect order of appearance in the introduction of the Model sections

* modified test description to make them latex compatible

* specialization for EnsembleForward and CustomSampler

* Apply suggestions from code review

addressed Congjian's comments

* Apply suggestions from code review

* Apply suggestions from code review

* Update ravenframework/Samplers/Sampler.py

* updated setuptools dep

* updated to simply ver 69

* added utility function as Congjian's request

* plot entity

* model order

* added starting models

* Apply suggestions from code review

* Update ravenframework/utils/graphStructure.py

---------

Co-authored-by: Congjian Wang - INL <congjian.wang@inl.gov>

doc/user_manual/model.tex

@@ -911,6 +911,14 @@ \section{Models}
   of Models (whose execution order is determined by the Input/Output relationships among them).
   If the relationships among the models evolve in a non-linear system, a Picard's Iteration scheme
   is employed.
+  \item \xmlNode{HybridModel} is a model aimed to combine reduced order models (ROMs) and 
+  any other high-fidelity Model (i.e., Code, ExternalModel). 
+  The ROMs will be trained based on the results from the high-fidelity model. 
+  The accuracy of the ROMs will be evaluated based on the cross validation scores, 
+  and the validity of the ROMs will be determined via some local validation metrics.
+  \item \xmlNode{LogicalModel} is a model aimed to execute ROMs, Codes and ExternalModels via a user 
+  provided control function. The control function utilizes the inputs generated by RAVEN and the control logic 
+  provided by the user to determine which model to execute.  
   \item \xmlNode{PostProcessor} is a container of all the actions that can
   manipulate and process a data object in order to extract key information,
   such as statistical quantities, clustering, etc.
@@ -1528,12 +1536,6 @@ \subsubsection{pickledModel}
 \end{lstlisting}
 }
 
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%%%%%         PostProcessor         %%%%%%%%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\input{postprocessor.tex}
-
 %
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %%%%%%  EnsembleModel  Model   %%%%%%
@@ -1996,3 +1998,8 @@ \subsection{HybridModel}
 %%%%%            Logical Model                %%%%%%%
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \input{logical_model.tex}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%         PostProcessor         %%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{postprocessor.tex}

ravenframework/Models/EnsembleModel.py

@@ -33,7 +33,7 @@
 #Internal Modules------------------------------------------------------------------------------------
 from .Dummy import Dummy
 from ..utils import utils, InputData
-from ..utils import graphStructure
+from ..utils.graphStructure import evaluateModelsOrder
 from ..Runners import Error as rerror
 #Internal Modules End--------------------------------------------------------------------------------
 
@@ -318,17 +318,17 @@ def initialize(self,runInfo,inputs,initDict=None):
         outputMatch.extend(match if match is not None else [])
       outputMatch = list(set(outputMatch))
       modelsToOutputModels[modelIn] = outputMatch
+    executionList, modelsGraph, _ = evaluateModelsOrder(modelsToOutputModels, acceptLoop=True, reverse=False, initialStartingModels=self.initialStartModels)
     # construct the ensemble model directed graph
-    self.ensembleModelGraph = graphStructure.graphObject(modelsToOutputModels)
+    self.ensembleModelGraph = modelsGraph #graphStructure.graphObject(modelsToOutputModels)
     # make some checks
     if not self.ensembleModelGraph.isConnectedNet():
       isolatedModels = self.ensembleModelGraph.findIsolatedVertices()
       self.raiseAnError(IOError, "Some models are not connected. Possible candidates are: "+' '.join(isolatedModels))
     # get all paths
-    allPath = self.ensembleModelGraph.findAllUniquePaths(self.initialStartModels)
     ###################################################
     # to be removed once executionList can be handled #
-    self.orderList = self.ensembleModelGraph.createSingleListOfVertices(allPath)
+    self.orderList = executionList
     self.raiseAMessage("Model Execution list: "+' -> '.join(self.orderList))
     ###################################################
     ###########################################################################################

ravenframework/OutStreams/PlotEntity.py

@@ -29,7 +29,6 @@
   matplotlib.use('Agg')
 
 
-
 class Plot(OutStreamEntity):
   """
     Handler for Plot implementations

ravenframework/Samplers/CustomSampler.py

@@ -164,7 +164,8 @@ def _localGenerateAssembler(self,initDict):
       else:
         mName = val
       self.funcDict[key] = fPointer(mName, initDict['Functions'][val])
-
+    # evaluate function order in custom sampler
+    self._evaluateFunctionsOrder()
     if 'Source' not in self.assemblerDict:
       self.raiseAnError(IOError, "No Source object has been found!")
 

ravenframework/Samplers/EnsembleForward.py

@@ -179,6 +179,9 @@ def _localGenerateAssembler(self, initDict):
         mName = val
       self.funcDict[key] = fPointer(mName, availableFunc[val])
 
+    # evaluate function order in custom sampler
+    self._evaluateFunctionsOrder()
+
   def localInitialize(self):
     """
       Initialize the EnsembleForward sampler. It calls the localInitialize method of all the Samplers defined in this input
@@ -247,11 +250,7 @@ def localGenerateInput(self, model, myInput):
     self.inputInfo['SamplerType'] = 'EnsembleForward'
 
     # Update dependent variables
-    for var in self.dependentSample:
-      test = self.funcDict[var].instance.evaluate(self.funcDict[var].methodName, self.inputInfo['SampledVars'])
-      for corrVar in var.split(","):
-        self.values[corrVar.strip()] = test
-        self.inputInfo['SampledVars'][corrVar.strip()] = test
+    self._functionalVariables()
 
   def flush(self):
     """

ravenframework/Samplers/Sampler.py

@@ -27,6 +27,7 @@
 from ..BaseClasses.InputDataUser import InputDataUser
 
 from ..utils import utils,randomUtils,InputData, InputTypes
+from ..utils.graphStructure import evaluateModelsOrder
 from ..BaseClasses import BaseEntity, Assembler
 
 class Sampler(utils.metaclass_insert(abc.ABCMeta, BaseEntity), Assembler, InputDataUser):
@@ -204,11 +205,16 @@ def __init__(self):
     self.distDict                      = {}          # Contains the instance of the distribution to be used, it is created every time the sampler is initialized. keys are the variable names
     self.funcDict                      = {}          # Mapping between variable name and the a 2-element namedtuple namedtuple('func', ['methodName', 'instance']) containing:
                                                      # element 0 (methodName): name of the method in the function to be be invoked. Either the default "evaluate", or the function name
+    self.variableFunctionExecutionList = []          # This is an ordered sequence of functional variable
+                                                     # (linked to functions) that need to be performed (in case of
+                                                     # interdependency). This list is always created. If no interdependence
+                                                     # is detected, the order is just random, otherwise the order is
+                                                     # determined through graph theory.
                                                      # element 1 (instance): instance of the function to be used, it is created every time the sampler is initialized.
     self.values                        = {}          # for each variable the current value {'var name':value}
     self.variableShapes                = {}          # stores the dimensionality of each variable by name, as tuple e.g. (2,3) for [[#,#,#],[#,#,#]]
     self.inputInfo                     = {}          # depending on the sampler several different type of keywarded information could be present only one is mandatory, see below
-    self.initSeed                      = None        # if not provided the seed is randomly generated at the istanciation of the sampler, the step can override the seed by sending in another seed
+    self.initSeed                      = None        # if not provided the seed is randomly generated at the initialization of the sampler, the step can override the seed by sending in another one
     self.inputInfo['SampledVars'     ] = self.values # this is the location where to get the values of the sampled variables
     self.inputInfo['SampledVarsPb'   ] = {}          # this is the location where to get the probability of the sampled variables
     self.inputInfo['crowDist']         = {}          # Stores a dictionary that contains the information to create a crow distribution.  Stored as a json object
@@ -277,6 +283,9 @@ def _generateDistributions(self, availableDist, availableFunc):
         mName = val
       self.funcDict[key] = fPointer(mName, availableFunc[val])
 
+    # evaluate function execution order
+    self._evaluateFunctionsOrder()
+
   def _localGenerateAssembler(self, initDict):
     """
       It is used for sending to the instanciated class, which is implementing the method, the objects that have been requested through "whatDoINeed" method
@@ -844,14 +853,41 @@ def _expandVectorVariables(self):
         baseVal = self.inputInfo['SampledVars'][var]
         self.inputInfo['SampledVars'][var] = np.ones(shape)*baseVal
 
+  def _evaluateFunctionsOrder(self):
+    """
+      Method to evaluate the function execution order using graph theory
+      The order is stored in self.variableFunctionExecutionList
+      @ In, None
+      @ Out, None
+    """
+    functionsToVariables = {}
+    for var in  self.funcDict:
+      outputMatch = []
+      functionInputs = self.funcDict[var].instance.parameterNames()
+      for inpVar in functionInputs:
+        # find functions that are linked to this inpVar
+        if inpVar in self.funcDict:
+          outputMatch.append(inpVar)
+      outputMatch = list(set(outputMatch))
+      functionsToVariables[var] =  outputMatch
+    executionList, variableFunctionsGraph, errMsg = evaluateModelsOrder(functionsToVariables, acceptLoop=False, reverse=True)
+    if errMsg is not None:
+      self.raiseAnError(*errMsg)
+    if  executionList:
+      self.variableFunctionExecutionList = executionList
+      self.raiseAMessage("Function Variables are interdependent")
+      self.raiseAMessage("Variable Evaluation and Function Execution list: "+
+                         ' -> '.join([f"variable:{var} | function: {self.funcDict[var].instance.name}"
+                                      for var in self.variableFunctionExecutionList]))
+
   def _functionalVariables(self):
     """
       Evaluates variables that are functions of other input variables.
       @ In, None
       @ Out, None
     """
     # generate the function variable values
-    for var in self.dependentSample:
+    for var in self.variableFunctionExecutionList:
       if self.inputInfo.get('batchMode',False):
         for b in range(self.inputInfo['batchInfo']['nRuns']):
           values = self.inputInfo['batchInfo']['batchRealizations'][b]['SampledVars']
@@ -1164,3 +1200,4 @@ def flush(self):
     self.auxcnt = 0
     self.distDict = {}
     self.funcDict = {}
+    self.variableFunctionExecutionList = []

ravenframework/utils/graphStructure.py

@@ -20,15 +20,50 @@
 from __future__ import division, print_function, absolute_import
 #----- end python 2 - 3 compatibility
 #External Modules------------------------------------------------------------------------------------
-import sys
 import itertools
 import copy
+from typing import Optional
 #External Modules End--------------------------------------------------------------------------------
 #Internal Modules------------------------------------------------------------------------------------
 from . import utils
 #Internal Modules End--------------------------------------------------------------------------------
 
 
+def evaluateModelsOrder(modelDict: dict, acceptLoop: Optional[bool] = True, reverse:  Optional[bool] = False,initialStartingModels: Optional[list] = []):
+  """
+    Utility method to evaluate the model/node execution order (From First(s) nodes till to the last
+    node(s) in the dictionary). The method uses graph theory for such evaluation.
+    The order, the graph object and (eventually) the error messages are returned in a tuple.
+    @ In, modelDict, dict, dictionary of models to outputs (e.g. {modelName1:[modelName2,modelName3],modelName2:[modelName4],..})
+    @ In, acceptLoop, bool, optional, should loops be accepted? Default: True
+    @ In, reverse, bool, optional, should the execution list be reversed? (Ie. First to Last or Last to First)
+    @ In, initialStartingModels, list, optional, initial starting models in case of "non-linear" connections
+    @ Out, executionList, list, model execution (ordered) list
+    @ Out, modelsGraph, graphObject, graph object
+    @ Out, errMsg, tuple or None, if tuple: el[0] -> Exception, el[1] -> error msg
+  """
+  errMsg = None # element 1 is Exception, element 2 is the error msg
+  if not len(modelDict):
+    return None, None, errMsg
+  modelsGraph = graphObject(modelDict)
+  # check for isolated models:
+  # isolated models are models that are not connected to other models
+  # consequentially thse models can be executed first (since no interdependency exists)
+  isolatedModels = modelsGraph.findIsolatedVertices()
+  executionList = isolatedModels
+  if len(isolatedModels) != len(modelDict):
+    if not acceptLoop and modelsGraph.isALoop():
+      errMsg = (IOError, "Models are interdependent but connections determined a loop of dependencies that "
+                        "is not supported in the system. Use EnsembleModel to solve such dependencies.")
+      return
+    allPath = modelsGraph.findAllUniquePaths(initialStartingModels)
+    executionList = modelsGraph.createSingleListOfVertices(allPath)
+    if reverse:
+      # the execution list is reversed becuase the created a graph above in reversed order (output to input)
+      executionList.reverse()
+    executionList = isolatedModels + executionList
+  return executionList, modelsGraph, errMsg
+
 class graphObject(object):
   """
     This is a class that crates a graph object.

tests/framework/VariablesFunctionsSamplersOptimizers/InterdependentVariableFunctions/interdependent_functions.py

@@ -0,0 +1,34 @@
+# Copyright 2017 Battelle Energy Alliance, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import imp
+import os
+# the functions below are imported from the respective files
+# this is aimed to show how multiple functions can be coded in the same file
+d_calc = imp.load_source('d_calc', os.path.join(os.path.dirname(os.path.abspath(__file__)),'../RedundantInputs/d_calc.py'))
+
+def raven_d_calc_f_a_c(ravenContainer):
+  return d_calc.evaluate(ravenContainer)
+
+def raven_e_calc_f_a_c(ravenContainer):
+  return ravenContainer.a + ravenContainer.c
+
+def raven_b_calc_f_a(ravenContainer):
+  return ravenContainer.a*1.2
+
+def raven_c_calc_f_b(ravenContainer):
+  return ravenContainer.b*1.2
+
+def raven_z_l_calc_f_a(ravenContainer):
+  return ravenContainer.a*1.2
+

tests/framework/VariablesFunctionsSamplersOptimizers/gold/InterdependentVariableFunctions/mc_out.csv

@@ -0,0 +1,3 @@
+a,b,c,d,e,z,l,f,g,h
+2.37409342951,2.84891211541,3.41869453849,8.11630024131,5.79278796799,2.84891211541,2.84891211541,6.76358203682,3.38917233333,17.1417897746
+4.35373093545,5.22447712254,6.26937254705,27.295161204,10.6231034825,5.22447712254,5.22447712254,22.7459674737,10.3522616,56.6023944473