Alfoa/go on ensemble model

framework/DataObjects/XDataSet.py

@@ -346,6 +346,8 @@ def getVars(self,subset=None):
       return self._outputs[:]
     elif subset == 'meta':
       return self._metavars[:]
+    elif subset == 'indexes':
+      return self.indexes[:]
     else:
       self.raiseAnError(KeyError,'Unrecognized subset choice: "{}"'.format(subset))
 

framework/Models/EnsembleModel.py

@@ -266,7 +266,7 @@ def initialize(self,runInfo,inputs,initDict=None):
       # attention: from now on, the values for the following dict with respect to 'Input' and 'Output' keys are changed
       # to the liss of input or output parameters names
       self.modelsDictionary[modelIn[2]]['Input' ]           = self.modelsDictionary[modelIn[2]]['TargetEvaluation'].getVars('input')
-      self.modelsDictionary[modelIn[2]]['Output']           = self.modelsDictionary[modelIn[2]]['TargetEvaluation'].getVars("output")
+      self.modelsDictionary[modelIn[2]]['Output']           = self.modelsDictionary[modelIn[2]]['TargetEvaluation'].getVars("output") + self.modelsDictionary[modelIn[2]]['TargetEvaluation'].getVars('indexes')
     # check if all the inputs passed in the step are linked with at least a model
     if not all(checkDictInputsUsage.values()):
       unusedFiles = ""
@@ -589,15 +589,6 @@ def _externalRun(self,inRun, jobHandler):
     identifier = inputKwargs.pop('prefix')
     tempOutputs = {}
     tempTargetEvaluations = {}
-    #holdOutputSpace = inputKwargs.values()[-1]['holdOutputSpace'] if 'holdOutputSpace' in inputKwargs.values()[-1] else None
-    # the sampler or optimizer wants to hold the result of
-    #modelsOnHold    = []
-    #holdCollector   = {}
-    #if holdOutputSpace is not None:
-    #  modelsOnHold = self._identifyModelsOnHold(holdOutputSpace[0])
-    #  for modelOnHold in modelsOnHold:
-    #    holdCollector[modelOnHold] = {'exportDict':self.tempOutputs['forHold'][holdOutputSpace[1]]['outs'][modelOnHold],'targetEvaluations':self.tempOutputs['forHold'][holdOutputSpace[1]]['targetEvaluations'][modelOnHold]} #         self.tempOutputs['forHold'][holdOutputSpace[1]][modelOnHold]
-    #    holdCollector[modelOnHold] = self.modelsDictionary[modelOnHold]['TargetEvaluation'].getRealizationGivenEvaluationID(holdOutputSpace[1])
 
     for modelIn in self.orderList:
       self.tempTargetEvaluations[modelIn].reset()
@@ -698,16 +689,16 @@ def _externalRun(self,inRun, jobHandler):
           # collect the target evaluation
           #if modelIn not in modelsOnHold:
           self.modelsDictionary[modelIn]['Instance'].collectOutput(finishedRun[0],tempTargetEvaluations[modelIn])
-          #else:
-          #  tempTargetEvaluations[modelIn] = holdCollector[modelIn]['targetEvaluations']
-          #dataSet = tempTargetEvaluations[modelIn].realization(index=0,unpackXArray=True)
+          ## FIXME: The call asDataset() is unuseful here. It must be done because otherwise the realization(...) method from collector
+          ## does not return the indexes values (TO FIX)
+          tempTargetEvaluations[modelIn].asDataset()
+          # get realization
           dataSet = tempTargetEvaluations[modelIn].realization(index=0,unpackXArray=True)
-          #FIXME: the following dict construction is a temporary solution since the realization method returns scalars if we have a PointSet
+          ##FIXME: the following dict construction is a temporary solution since the realization method returns scalars if we have a PointSet
           dataSet = {key:np.atleast_1d(dataSet[key]) for key in dataSet}
           responseSpace         = dataSet
-          #inputSpace            = tempTargetEvaluations[modelIn].getParametersValues('inputs', nodeId = 'RecontructEnding')
           typeOutputs[modelCnt] = tempTargetEvaluations[modelIn].type
-          gotOutputs[modelCnt]  = {key: dataSet[key] for key in tempTargetEvaluations[modelIn].getVars("output")}   # responseSpace if typeOutputs[modelCnt] != 'HistorySet' else responseSpace.values()[-1]
+          gotOutputs[modelCnt]  = {key: dataSet[key] for key in tempTargetEvaluations[modelIn].getVars("output")+tempTargetEvaluations[modelIn].getVars("indexes")}
 
           #store the results in return dictionary
           # store the metadata
@@ -745,11 +736,3 @@ def _externalRun(self,inRun, jobHandler):
           break
     returnEvaluation = returnDict, tempTargetEvaluations, tempOutputs
     return returnEvaluation
-
-  #def acceptHoldOutputSpace(self):
-  #  """
-  #    This method returns True if a certain output space can be kept on hold (so far, just the EnsembelModel can do that)
-  #    @ In, None
-  #    @ Out, acceptHoldOutputSpace, bool, True if a certain output space can be kept on hold
-  #  """
-  #  return True

framework/Models/HybridModel.py

@@ -190,7 +190,7 @@ def initialize(self,runInfo,inputs,initDict=None):
     self.cvInstance = self.retrieveObjectFromAssemblerDict('CV', self.cvInstance)
     self.cvInstance.initialize(runInfo, inputs, initDict)
     self.targetEvaluationInstance = self.retrieveObjectFromAssemblerDict('TargetEvaluation', self.targetEvaluationInstance)
-    if not self.targetEvaluationInstance.isItEmpty():
+    if len(self.targetEvaluationInstance):
       self.raiseAWarning("The provided TargetEvaluation data object is not empty, the existing data will also be used to train the ROMs!")
       self.existTrainSize = len(self.targetEvaluationInstance)
     self.tempTargetEvaluation = copy.deepcopy(self.targetEvaluationInstance)
@@ -204,8 +204,8 @@ def initialize(self,runInfo,inputs,initDict=None):
       romInfo['Instance'] = self.retrieveObjectFromAssemblerDict('ROM', romName)
       if romInfo['Instance']  is None:
         self.raiseAnError(IOError, 'ROM XML block needs to be inputted!')
-    modelInputs = self.targetEvaluationInstance.getParaKeys("inputs")
-    modelOutputs = self.targetEvaluationInstance.getParaKeys("outputs")
+    modelInputs = self.targetEvaluationInstance.getVars("input")
+    modelOutputs = self.targetEvaluationInstance.getVars("output")
     modelName = self.modelInstance.name
     totalRomOutputs = []
     for romInfo in self.romsDictionary.values():
@@ -365,13 +365,17 @@ def isRomConverged(self, outputDict):
       @ Out, converged, bool, True if the rom is converged
     """
     converged = True
-    for targetName, metricInfo in outputDict.items():
-      if len(metricInfo.keys()) > 1:
+    # very temporary solution
+    exploredTargets = []
+    for cvKey, metricValues in outputDict.items():
+      #for targetName, metricInfo in outputDict.items():
+      # very temporary solution
+      info = self.cvInstance.interface._returnCharacteristicsOfCvGivenOutputName(cvKey)
+      if info['targetName'] in exploredTargets:
         self.raiseAnError(IOError, "Multiple metrics are used in cross validation '", self.cvInstance.name, "'. Currently, this can not be processed by the HybridModel '", self.name, "'!")
-      for metricName, metricValues in metricInfo.items():
-        name = self.cvInstance.interface.metricsDict.keys()[0]
-        metricType = metricName[len(name)+1:]
-        converged = self.checkErrors(metricType, metricValues)
+      exploredTargets.append(info['targetName'])
+      name = self.cvInstance.interface.metricsDict.keys()[0]
+      converged = self.checkErrors(info['metricType'], metricValues)
     return converged
 
   def checkErrors(self, metricType, metricResults):
@@ -381,7 +385,7 @@ def checkErrors(self, metricType, metricResults):
       @ In, metricResults, list or dict
       @ Out, converged, bool, True if the metric outputs are less than the tolerance
     """
-    if type(metricResults) == list:
+    if type(metricResults) == list or isinstance(metricResults,np.ndarray):
       errorList = np.atleast_1d(metricResults)
     elif type(metricResults) == dict:
       errorList = np.atleast_1d(metricResults.values())
@@ -602,7 +606,7 @@ def evaluateSample(self, myInput, samplerType, kwargs):
       @ In, samplerType, string, is the type of sampler that is calling to generate a new input
       @ In, kwargs, dict,  is a dictionary that contains the information coming from the sampler,
         a mandatory key is the sampledVars'that contains a dictionary {'name variable':value}
-      @ Out, returnValue, dict, This holds the output information of the evaluated sample.
+      @ Out, rlz, dict, This holds the output information of the evaluated sample.
     """
     self.raiseADebug("Evaluate Sample")
     kwargsKeys = kwargs.keys()
@@ -611,8 +615,13 @@ def evaluateSample(self, myInput, samplerType, kwargs):
     jobHandler = kwargs['jobHandler']
     newInput = self.createNewInput(myInput, samplerType, **kwargsToKeep)
     ## Unpack the specifics for this class, namely just the jobHandler
-    returnValue = (newInput,self._externalRun(newInput,jobHandler))
-    return returnValue
+    result = self._externalRun(newInput,jobHandler)
+    # assure rlz has all metadata
+    rlz = dict((var,np.atleast_1d(kwargsToKeep[var])) for var in kwargsToKeep.keys())
+    # update rlz with input space from inRun and output space from result
+    rlz.update(dict((var,np.atleast_1d(kwargsToKeep['SampledVars'][var] if var in kwargs['SampledVars'] else result[var])) for var in set(result.keys()+kwargsToKeep['SampledVars'].keys())))
+
+    return rlz
 
   def _externalRun(self,inRun, jobHandler):
     """
@@ -653,7 +662,7 @@ def _externalRun(self,inRun, jobHandler):
           if isinstance(evaluation, Runners.Error):
             self.raiseAnError(RuntimeError, "The job identified by "+finishedRun.identifier+" failed!")
           # collect output in temporary data object
-          tempExportDict = self.createExportDictionaryFromFinishedJob(finishedRun, False)
+          tempExportDict = evaluation #self.createExportDictionaryFromFinishedJob(finishedRun, False)
           exportDict = self.__mergeDict(exportDict, tempExportDict)
         if jobHandler.areTheseJobsFinished(uniqueHandler=uniqueHandler):
           self.raiseADebug("Jobs with uniqueHandler ", uniqueHandler, "are collected!")
@@ -680,7 +689,7 @@ def _externalRun(self,inRun, jobHandler):
       if isinstance(evaluation, Runners.Error):
         self.raiseAnError(RuntimeError, "The model "+self.modelInstance.name+" identified by "+finishedRun[0].identifier+" failed!")
       # collect output in temporary data object
-      exportDict = self.modelInstance.createExportDictionaryFromFinishedJob(finishedRun[0], True)
+      exportDict = evaluation #self.modelInstance.createExportDictionaryFromFinishedJob(finishedRun[0], True)
       self.raiseADebug("Create exportDict")
     # used in the collectOutput
     exportDict['useROM'] = useROM
@@ -696,17 +705,19 @@ def collectOutput(self,finishedJob,output):
     evaluation = finishedJob.getEvaluation()
     if isinstance(evaluation, Runners.Error):
       self.raiseAnError(RuntimeError,"Job " + finishedJob.identifier +" failed!")
-    exportDict = evaluation[1]
-    useROM = exportDict['useROM']
+    #exportDict = evaluation[1]
+    #exportDict = evaluation[1]
+    useROM = evaluation['useROM']
     try:
       jobIndex = self.tempOutputs['uncollectedJobIds'].index(finishedJob.identifier)
       self.tempOutputs['uncollectedJobIds'].pop(jobIndex)
     except ValueError:
       jobIndex = None
     if jobIndex is not None and not useROM:
-      self.collectOutputFromDict(exportDict, self.tempTargetEvaluation)
+      self.tempTargetEvaluation.addRealization(evaluation)
+      #self.collectOutputFromDict(exportDict, self.tempTargetEvaluation)
       self.raiseADebug("ROM is invalid, collect ouptuts of Model with job identifier: {}".format(finishedJob.identifier))
-    Dummy.collectOutput(self, finishedJob, output, options = {'exportDict':exportDict})
+    Dummy.collectOutput(self, finishedJob, output )#, options = {'exportDict':exportDict})
 
   def __mergeDict(self,exportDict, tempExportDict):
     """

framework/PostProcessorFunctions/TypicalHistoryFromHistorySet.py

@@ -232,7 +232,7 @@ def run(self,inputDic):
         typicalDataHistories[index][feature] = subseqData[index][feature][smallestDeltaIndex,:]
     # now collapse the data into the typical history
     typicalData = {}
-    typicalData[self.pivotParameter] = np.concatenate(list(typicalDataHistories[index][self.pivotParameter] for index in range(numParallelSubsequences)))  
+    typicalData[self.pivotParameter] = np.concatenate(list(typicalDataHistories[index][self.pivotParameter] for index in range(numParallelSubsequences)))
     for feature in self.features:
       typicalData[feature] = np.concatenate(list(typicalDataHistories[index][feature] for index in range(numParallelSubsequences)))
     # sanity check, should probably be skipped for efficiency, as it looks like a debugging tool
@@ -253,7 +253,7 @@ def run(self,inputDic):
       outputDict['data'][var][0] = inputDict[var][0]
     outputDict['dims']={}
     for var in self.features:
-      outputDict['dims'][var]=[self.pivotParameter] 
+      outputDict['dims'][var]=[self.pivotParameter]
     return outputDict
 
   def __computeECDF(self, data, binEdgesIn):

framework/PostProcessors/CrossValidation.py

@@ -256,6 +256,19 @@ def inputToInternal(self, currentInp, full = False):
     newInputs = newInput, cvEstimator
     return newInputs
 
+  #FIXME: Temporary method. Need to be rethought when the new Hybrid Model is developed
+  def _returnCharacteristicsOfCvGivenOutputName(self,outputName):
+    """
+      Method to return the metric name, type and target name given the output name
+      @ In, outputName, str, the output name
+      @ Out, info, dict, the dictionary containing the info
+    """
+    assert(len(outputName.split("_")) == 3)
+    info = {}
+    _, info['metricName'], info['targetName']  = outputName.split("_")
+    info['metricType'] = self.metricsDict[info['metricName']].metricType
+    return info
+
   def __generateTrainTestInputs(self, inputDict, trainIndex, testIndex):
     """
       Genenerate train and test set based on the given train index and test index

tests/framework/CodeInterfaceTests/raven_running_raven_internal_models/test_rom_trainer.xml

@@ -148,7 +148,7 @@
     </PointSet>
     <PointSet inputTs="2" name="Pointset_from_database_for_rom_trainer">
       <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>CladTempThreshold</Output>
+      <Output>CladTempThreshold,time</Output>
     </PointSet>
     <PointSet historyName="1" inputTs="2" name="data_for_sampling_empty_at_begin">
       <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
@@ -158,17 +158,17 @@
       <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
       <Output>OutputPlaceHolder</Output>
     </PointSet>
-    <PointSet inputTs="2" name="outputMontecarloRom">
+    <PointSet name="outputMontecarloRom">
       <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
       <Output>CladTempThreshold</Output>
     </PointSet>
     <HistorySet name="outputMontecarloRomHS">
         <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-        <Output>CladTempThreshold</Output>
+        <Output>CladTempThreshold,time</Output>
     </HistorySet>
     <PointSet inputTs="2" name="outputMontecarloRomND">
       <Input>DeltaTimeScramToAux,DG1recoveryTime</Input>
-      <Output>CladTempThreshold</Output>
+      <Output>CladTempThreshold,time</Output>
     </PointSet>
   </DataObjects>
 

tests/framework/ensembleModelTests/test_ensemble_model_linear_threading_with_time_series.xml

@@ -220,7 +220,10 @@
     </PointSet>
     <HistorySet name="thermalConductivityComputationContainer">
         <Input>leftTemperature,rightTemperature</Input>
-        <Output>k,timeFromFirstModel</Output>
+        <Output>k</Output>
+        <options>
+            <pivotParameter>timeFromFirstModel</pivotParameter>
+        </options>
     </HistorySet>
   </DataObjects>
 

tests/framework/gold/output_check/fullprint_HistorySet.xml

@@ -7,7 +7,7 @@
     </dims>
     <general>
       <inputs>var1,var2</inputs>
-      <pointwise_meta>ProbabilityWeight-var1,ProbabilityWeight-var2,ProbabilityWeight,prefix,PointProbability</pointwise_meta>
+      <pointwise_meta>ProbabilityWeight,ProbabilityWeight-var1,prefix,PointProbability,ProbabilityWeight-var2</pointwise_meta>
       <outputs>sine,square,cosine</outputs>
       <sampleTag>RAVEN_sample_ID</sampleTag>
     </general>

tests/framework/gold/output_check/fullprint_Pointset.xml

@@ -2,7 +2,7 @@
   <DataSet type="Static">
     <general>
       <inputs>var1,var2</inputs>
-      <pointwise_meta>ProbabilityWeight-var1,ProbabilityWeight-var2,ProbabilityWeight,prefix,PointProbability</pointwise_meta>
+      <pointwise_meta>ProbabilityWeight,ProbabilityWeight-var1,prefix,PointProbability,ProbabilityWeight-var2</pointwise_meta>
       <outputs>time,sine,square,cosine</outputs>
       <sampleTag>RAVEN_sample_ID</sampleTag>
     </general>

tests/framework/hybridModel/gold/romTrainRom/metaModelOutputTestDump.csv

@@ -1,41 +1,41 @@
-rightTemperature,leftTemperature,k
-1261.31072723,769.597955763,0.0322050521833
-1174.50459336,1525.24138517,0.0322050521833
-1571.16098317,1648.6958241,0.0322050521833
-1269.60940817,1125.61331285,0.0322050521833
-997.315857489,500.262433477,0.0322050521833
-511.750896278,1179.76339298,0.0322050521833
-1395.79177603,1118.11303082,0.0322050521833
-1335.84571491,1425.07522938,0.0322050521833
-1010.43601057,1138.74257958,0.0322050521833
-1350.63522599,1347.57975741,0.0322050521833
-1054.06268615,726.586376928,0.0322050521833
-1620.82760677,1646.20702526,0.0322050521833
-1667.87949651,1187.79552828,0.0322050521833
-1603.45520049,783.539288371,0.0322050521833
-1508.78469344,1122.47902309,0.0322050521833
-1130.525271,1019.85670256,0.0322050521833
-1002.19029516,669.596191442,0.0322050521833
-1032.38404806,1446.77175976,0.0322050521833
-1310.60222658,1144.03216956,0.0322050521833
-1199.90569556,1367.12919121,0.0322050521833
-601.558282064,1015.01618226,0.0322050521833
-1593.72978217,1632.60342161,0.0322050521833
-861.623179438,1310.71645683,0.0322050521833
-1161.65727038,664.054127355,0.0322050521833
-908.710236011,1200.45852817,0.0322050521833
-914.092090077,1361.69643245,0.0322050521833
-1519.01430306,900.562822167,0.0322050521833
-1667.23133288,566.834234276,0.0322050521833
-1006.46963169,696.796910045,0.0322050521833
-1051.59998521,1364.35290008,0.0322050521833
-919.53782598,872.210895822,0.0322050521833
-599.860508018,1672.43886627,0.0322050521833
-1415.49732641,1572.65856552,0.0322050521833
-1378.3528113,873.482695961,0.0298833181459
-1692.56615657,1156.06354164,0.0301285039345
-1362.96807808,520.29037057,0.0322050521833
-766.296345337,1333.69837553,0.0320933195934
-1496.13095052,1258.43668635,0.0295905227099
-1042.90632357,1526.60560026,0.0322050521833
-1676.54607258,989.114559183,0.0316390586337
+leftTemperature,rightTemperature,k
+769.597955763,1261.31072723,0.0322050521833
+1525.24138517,1174.50459336,0.0322050521833
+1648.6958241,1571.16098317,0.0322050521833
+1125.61331285,1269.60940817,0.0322050521833
+500.262433477,997.315857489,0.0322050521833
+1179.76339298,511.750896278,0.0322050521833
+1118.11303082,1395.79177603,0.0322050521833
+1425.07522938,1335.84571491,0.0322050521833
+1138.74257958,1010.43601057,0.0322050521833
+1347.57975741,1350.63522599,0.0322050521833
+726.586376928,1054.06268615,0.0322050521833
+1646.20702526,1620.82760677,0.0322050521833
+1187.79552828,1667.87949651,0.0322050521833
+783.539288371,1603.45520049,0.0322050521833
+1122.47902309,1508.78469344,0.0322050521833
+1019.85670256,1130.525271,0.0322050521833
+669.596191442,1002.19029516,0.0322050521833
+1446.77175976,1032.38404806,0.0322050521833
+1144.03216956,1310.60222658,0.0322050521833
+1367.12919121,1199.90569556,0.0322050521833
+1015.01618226,601.558282064,0.0322050521833
+1632.60342161,1593.72978217,0.0322050521833
+1310.71645683,861.623179438,0.0322050521833
+664.054127355,1161.65727038,0.0322050521833
+1200.45852817,908.710236011,0.0322050521833
+1361.69643245,914.092090077,0.0322050521833
+900.562822167,1519.01430306,0.0322050521833
+566.834234276,1667.23133288,0.0322050521833
+696.796910045,1006.46963169,0.0322050521833
+1364.35290008,1051.59998521,0.0322050521833
+872.210895822,919.53782598,0.0322050521833
+1672.43886627,599.860508018,0.0322050521833
+1572.65856552,1415.49732641,0.0322050521833
+873.482695961,1378.3528113,0.0298833181459
+1156.06354164,1692.56615657,0.0301285039345
+520.29037057,1362.96807808,0.0322050521833
+1333.69837553,766.296345337,0.0320933195934
+1258.43668635,1496.13095052,0.0295905227099
+1526.60560026,1042.90632357,0.0322050521833
+989.114559183,1676.54607258,0.0316390586337

tests/framework/hybridModel/test_code.xml

@@ -123,6 +123,7 @@
     <Print name="samples">
       <type>csv</type>
       <source>samples</source>
+      <what>input, output</what>
     </Print>
   </OutStreams>
 

tests/framework/hybridModel/test_hybrid_model.xml

@@ -103,6 +103,7 @@
     <Print name="metaModelOutputTestDump">
       <type>csv</type>
       <source>metaModelOutputTest</source>
+      <what>input, output</what>
     </Print>
   </OutStreams>