Adding a KerasMLPRegression class to go with the KerasMLPClassifier. (#…

…1707)

* Adding a KerasMLPRegression class to go with the KerasMLPClassifier.

This allows regression to be done for non-timeseries data.

* Adding rel_err because of differences between computers.

doc/user_manual/kerasROM.tex

@@ -568,9 +568,10 @@ \subsubsection{TensorFlow-Keras Deep Neural Networks}
     and an upper bound.
 \end{itemize}
 
-%%%%% ROM Model - KerasMLPClassifier  %%%%%%%
-\paragraph{KerasMLPClassifier}
+%%%%% ROM Model - KerasMLPClassifier and KerasMLPRegression  %%%%%%%
+\paragraph{KerasMLPClassifier and KerasMLPRegression}
 \label{KerasMLPClassifier}
+\label{KerasMLPRegression}
 
 Multi-Layer Perceptron (MLP) (or Artificial Neural Network - ANN), a class of feedforward
 ANN, can be viewed as a logistic regression classifier where input is first transformed
@@ -584,18 +585,18 @@ \subsubsection{TensorFlow-Keras Deep Neural Networks}
 relationships. The extra layers enable composition of features from lower layers, potentially
 modeling complex data with fewer units than a similarly performing shallow network.
 
-\zNormalizationPerformed{KerasMLPClassifier}
+\zNormalizationPerformed{KerasMLPClassifier \textup{and} KerasMLPRegression}
 
 In order to use this ROM, the \xmlNode{ROM} attribute \xmlAttr{subType} needs to
-be \xmlString{KerasMLPClassifier} (see the example below). This model can be initialized with
+be \xmlString{KerasMLPClassifier} or \xmlString{KerasMLPRegression} (see the examples below). This model can be initialized with
 the following layers:
 
 \begin{itemize}
   \DenseLayer
   \DropoutLayer
 \end{itemize}
 
-\textbf{Example:}
+\textbf{KerasMLPClassifier Example:}
 \begin{lstlisting}[style=XML,morekeywords={name,subType}]
 <Simulation>
   ...
@@ -641,6 +642,35 @@ \subsubsection{TensorFlow-Keras Deep Neural Networks}
 </Simulation>
 \end{lstlisting}
 
+\textbf{KerasMLPRegression Example:}
+\begin{lstlisting}[style=XML,morekeywords={name,subType}]
+<Simulation>
+  ...
+  <Models>
+    <ROM name="modelUnderTest" subType="KerasMLPRegression">
+      <Features>x1,x2,x3,x4,x5,x6,x7,x8</Features>
+      <Target>y</Target>
+      <loss>mean_squared_error</loss>
+      <batch_size>10</batch_size>
+      <epochs>60</epochs>
+      <plot_model>False</plot_model>
+      <validation_split>0.25</validation_split>
+      <random_seed>1986</random_seed>
+      <Dense name="layer1">
+          <dim_out>30</dim_out>
+      </Dense>
+      <Dense name="layer2">
+          <dim_out>12</dim_out>
+      </Dense>
+      <Dense name="outLayer">
+      </Dense>
+      <layer_layout>layer1, layer2, outLayer</layer_layout>
+    </ROM>
+  </Models>
+  ...
+</Simulation>
+\end{lstlisting}
+
 %%%%% ROM Model - KerasConvNetClassifier  %%%%%%%
 \paragraph{KerasConvNetClassifier}
 \label{KerasClassifier}

framework/SupervisedLearning/Factory.py

@@ -36,6 +36,7 @@
 
 ## Tensorflow-Keras Neural Network Models
 from .KerasMLPClassifier import KerasMLPClassifier
+from .KerasMLPRegression import KerasMLPRegression
 from .KerasConvNetClassifier import KerasConvNetClassifier
 from .KerasLSTMClassifier import KerasLSTMClassifier
 from .KerasLSTMRegression import KerasLSTMRegression

framework/SupervisedLearning/KerasBase.py

@@ -1893,7 +1893,7 @@ def __init__(self):
 
     self.kerasLayersList = functools.reduce(lambda x,y: x+y, list(self.kerasDict.values()))
 
-    self.kerasROMsList = ['KerasMLPClassifier', 'KerasConvNetClassifier', 'KerasLSTMClassifier', 'KerasLSTMRegression']
+    self.kerasROMsList = ['KerasMLPClassifier', 'KerasMLPRegression', 'KerasConvNetClassifier', 'KerasLSTMClassifier', 'KerasLSTMRegression']
 
     if len(self.availOptimizer) == 0:
       # stochastic gradient descent optimizer, includes support for momentum,learning rate decay, and Nesterov momentum

framework/SupervisedLearning/KerasLSTMRegression.py

@@ -19,6 +19,8 @@
 """
 #External Modules------------------------------------------------------------------------------------
 import numpy as np
+import utils.importerUtils
+tf = utils.importerUtils.importModuleLazyRenamed("tf", globals(), "tensorflow")
 ######
 #Internal Modules------------------------------------------------------------------------------------
 from .KerasRegression import KerasRegression
@@ -81,6 +83,26 @@ def _checkLayers(self):
           self.initOptionDict[layerName]['return_sequences'] = True
           self.raiseAWarning('return_sequences is resetted to True for layer',layerName)
 
+  def _getFirstHiddenLayer(self, layerInstant, layerSize, layerDict):
+    """
+      Creates the first hidden layer
+      @ In, layerInstant, class, layer type from tensorflow.python.keras.layers
+      @ In, layerSize, int, nodes in layer
+      @ In, layerDict, dict, layer details
+      @ Out, layer, tensorflow.python.keras.layers, new layer
+    """
+    return layerInstant(layerSize,input_shape=[None,self.featv.shape[-1]], **layerDict)
+
+  def _getLastLayer(self, layerInstant, layerDict):
+    """
+      Creates the last layer
+      @ In, layerInstant, class, layer type from tensorflow.python.keras.layers
+      @ In, layerSize, int, nodes in layer
+      @ In, layerDict, dict, layer details
+      @ Out, layer, tensorflow.python.keras.layers, new layer
+    """
+    return tf.keras.layers.TimeDistributed(layerInstant(len(self.targv),**layerDict))
+
   def _preprocessInputs(self,featureVals):
     """
       Perform input feature values before sending to ROM prediction

framework/SupervisedLearning/KerasMLPRegression.py

@@ -0,0 +1,74 @@
+# Copyright 2017 Battelle Energy Alliance, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+  Created on 3-Nov-2021
+
+  @author: cogljj
+  module for Multi-layer perceptron regression
+"""
+#Internal Modules------------------------------------------------------------------------------------
+from .KerasRegression import KerasRegression
+#Internal Modules End--------------------------------------------------------------------------------
+
+class KerasMLPRegression(KerasRegression):
+  """
+    Multi-layer perceptron regressor constructed using Keras API in TensorFlow
+  """
+  info = {'problemtype':'regression', 'normalize':True}
+
+  @classmethod
+  def getInputSpecification(cls):
+    """
+      Method to get a reference to a class that specifies the input data for
+      class cls.
+      @ In, cls, the class for which we are retrieving the specification
+      @ Out, inputSpecification, InputData.ParameterInput, class to use for
+        specifying input of cls.
+    """
+    specs = super().getInputSpecification()
+    specs.description = r"""Multi-Layer Perceptron (MLP) (or Artificial Neural Network - ANN), a class of feedforward
+        ANN, can be viewed as a logistic regression where input is first transformed
+        using a non-linear transformation. This transformation probjects the input data into a
+        space where it becomes linearly separable. This intermediate layer is referred to as a
+        \textbf{hidden layer}. An MLP consists of at least three layers of nodes. Except for the
+        input nodes, each node is a neuron that uses a nonlinear \textbf{activation function}. MLP
+        utilizes a suppervised learning technique called \textbf{Backpropagation} for training.
+        Generally, a single hidden layer is sufficient to make MLPs a universal approximator.
+        However, many hidden layers, i.e. deep learning, can be used to model more complex nonlinear
+        relationships. The extra layers enable composition of features from lower layers, potentially
+        modeling complex data with fewer units than a similarly performing shallow network.
+        \\
+        \zNormalizationPerformed{KerasMLPRegression}
+        \\
+        In order to use this ROM, the \xmlNode{ROM} attribute \xmlAttr{subType} needs to
+        be \xmlString{KerasMLPRegression}"""
+    return specs
+
+  def __init__(self):
+    """
+      A constructor that will appropriately intialize a supervised learning object
+      @ In, None
+      @ Out, None
+    """
+    super().__init__()
+    self.printTag = 'KerasMLPRegression'
+    self.allowedLayers = self.basicLayers
+
+  def _handleInput(self, paramInput):
+    """
+      Function to handle the common parts of the model parameter input.
+      @ In, paramInput, InputData.ParameterInput, the already parsed input.
+      @ Out, None
+    """
+    super()._handleInput(paramInput)

framework/SupervisedLearning/KerasRegression.py

@@ -71,7 +71,7 @@ def _getFirstHiddenLayer(self, layerInstant, layerSize, layerDict):
       @ In, layerDict, dict, layer details
       @ Out, layer, tensorflow.python.keras.layers, new layer
     """
-    return layerInstant(layerSize,input_shape=[None,self.featv.shape[-1]], **layerDict)
+    return layerInstant(layerSize,input_shape=[self.featv.shape[-1]], **layerDict)
 
   def _getLastLayer(self, layerInstant, layerDict):
     """
@@ -81,15 +81,15 @@ def _getLastLayer(self, layerInstant, layerDict):
       @ In, layerDict, dict, layer details
       @ Out, layer, tensorflow.python.keras.layers, new layer
     """
-    return tf.keras.layers.TimeDistributed(layerInstant(len(self.targv),**layerDict))
+    return layerInstant(self.targv.shape[-1],**layerDict)
 
   def _getTrainingTargetValues(self, names, values):
     """
       Gets the target values to train with, which differs depending
       on if this is a regression or classifier.
       @ In, names, list of names
       @ In, values, list of values
-      @ Out, targetValues, numpy.ndarray of shape (numSamples, numTimesteps, numFeatures)
+      @ Out, targetValues, numpy.ndarray of shape (numSamples, numTimesteps, numFeatures) or shape (numSamples, numFeatures)
     """
     # Features must be 3d i.e. [numSamples, numTimeSteps, numFeatures]
 
@@ -98,11 +98,17 @@ def _getTrainingTargetValues(self, names, values):
         self.raiseAnError(IOError,'The target '+target+' is not in the training set')
 
     firstTarget = values[names.index(self.target[0])]
-    targetValues = np.zeros((len(firstTarget), len(firstTarget[0]),
-                             len(self.target)))
-    for i, target in enumerate(self.target):
-      self._localNormalizeData(values, names, target)
-      targetValues[:, :, i] = self._scaleToNormal(values[names.index(target)], target)
+    if type(firstTarget) == type(np.array(1)) and len(firstTarget.shape) == 1:
+      targetValues = np.zeros((len(firstTarget), len(self.target)))
+      for i, target in enumerate(self.target):
+        self._localNormalizeData(values, names, target)
+        targetValues[:, i] = self._scaleToNormal(values[names.index(target)], target)
+    else:
+      targetValues = np.zeros((len(firstTarget), len(firstTarget[0]),
+                               len(self.target)))
+      for i, target in enumerate(self.target):
+        self._localNormalizeData(values, names, target)
+        targetValues[:, :, i] = self._scaleToNormal(values[names.index(target)], target)
     return targetValues
 
 
@@ -145,7 +151,6 @@ def evaluate(self,edict):
           featureValuesShape = fval.shape
         if featureValuesShape != fval.shape:
           self.raiseAnError(IOError,'In training set, the number of values provided for feature '+feat+' are not consistent to other features!')
-        self._localNormalizeData(values,names,feat)
         fval = self._scaleToNormal(fval, feat)
         featureValues.append(fval)
       else:
@@ -154,7 +159,9 @@ def evaluate(self,edict):
 
     result = self.__evaluateLocal__(featureValues)
     pivotParameter = self.pivotID
-    if type(edict[pivotParameter]) == type([]):
+    if pivotParameter not in edict:
+      pass #don't need to do anything
+    elif type(edict[pivotParameter]) == type([]):
       #XXX this should not be needed since sampler should just provide the numpy array.
       #Currently the CustomSampler provides all the pivot parameter values instead of the current one.
       self.raiseAWarning("Adjusting pivotParameter because incorrect type provided")
@@ -175,5 +182,8 @@ def __evaluateLocal__(self,featureVals):
     prediction = {}
     outcome = self._ROM.predict(featureVals)
     for i, target in enumerate(self.target):
-      prediction[target] = self._invertScaleToNormal(outcome[0, :, i], target)
+      if len(outcome.shape) == 3:
+        prediction[target] = self._invertScaleToNormal(outcome[0, :, i], target)
+      else:
+        prediction[target] = self._invertScaleToNormal(outcome[0, i], target)
     return prediction