Adapt HERON to use ravenframework (#140)

* Adapt HERON to use ravenframework

* Fix heron executable script

* Modify get_raven_loc

* Retrain linear_rom.pk

* Fix more import statements

* Regold Labels test due to idaholab/raven#1780 PR

* Add Analytic.xslsx to Cashflows test

* Regold Cashflow tests

* Regold Multirun Opt

heron

@@ -12,8 +12,7 @@ fi
 HERON_DIR=$(cd $HERON_DIRNAME; pwd)
 
 # identify RAVEN location
-RAVENFW_DIR=$(python $HERON_DIRNAME/src/_utils.py get_raven_loc)
-RAVEN_DIR=$(cd $RAVENFW_DIR/../; pwd)
+RAVEN_DIR=$(python $HERON_DIRNAME/src/_utils.py get_raven_loc)
 echo " ... RAVEN located at \"${RAVEN_DIR}\""
 # activate conda environment
 echo " ... Activating CONDA environment ..."
@@ -48,4 +47,4 @@ then
 else
   echo ' ... Errors were encountered while running HERON.'
 fi
-exit $SUCCESS
+exit $SUCCESS

src/Cases.py

@@ -25,7 +25,7 @@
 import _utils as hutils
 framework_path = hutils.get_raven_loc()
 sys.path.append(framework_path)
-from utils import InputData, InputTypes, xmlUtils
+from ravenframework.utils import InputData, InputTypes, xmlUtils
 
 class Case(Base):
   """
@@ -677,7 +677,7 @@ def _load_template(self):
     template_name = 'template_driver'
     # import template module
     sys.path.append(heron_dir)
-    module = importlib.import_module('templates.{}'.format(template_name))
+    module = importlib.import_module('templates.{}'.format(template_name), package="HERON")
     # load template, perform actions
     template_class = module.Template(messageHandler=self.messageHandler)
     template_class.loadTemplate(template_dir)

src/Components.py

@@ -17,7 +17,7 @@
 
 framework_path = hutils.get_raven_loc()
 sys.path.append(framework_path)
-from utils import InputData, xmlUtils,InputTypes
+from ravenframework.utils import InputData, xmlUtils,InputTypes
 
 # TODO can we use EntityFactory from RAVEN?
 def factory(xml, method='sweep'):

src/DispatchPlot.py

@@ -10,7 +10,7 @@
 import matplotlib.pyplot as plt
 from typing import List, Dict
 
-from PluginBaseClasses.OutStreamPlotPlugin import PlotPlugin, InputTypes, InputData
+from ravenframework.PluginBaseClasses.OutStreamPlotPlugin import PlotPlugin, InputTypes, InputData
 
 
 # Matplotlib Global Settings

src/Economics.py

@@ -14,7 +14,7 @@
 import _utils as hutils
 framework_path = hutils.get_raven_loc()
 sys.path.append(framework_path)
-from utils import InputData, xmlUtils,InputTypes
+from ravenframework.utils import InputData, xmlUtils,InputTypes
 
 
 class CashFlowUser:

src/Placeholders.py

@@ -14,10 +14,10 @@
 
 FRAMEWORK_PATH = hutils.get_raven_loc()
 sys.path.append(FRAMEWORK_PATH)
-from utils import InputData, InputTypes, utils, xmlUtils
+from ravenframework.utils import InputData, InputTypes, utils, xmlUtils
 sys.path.pop()
 
-sys.path.append(os.path.join(FRAMEWORK_PATH, os.pardir, 'scripts'))
+sys.path.append(os.path.join(FRAMEWORK_PATH, 'scripts'))
 from externalROMloader import ravenROMexternal
 sys.path.pop()
 

src/ValuedParamHandler.py

@@ -7,14 +7,16 @@
 from __future__ import unicode_literals, print_function
 import os
 import sys
+
 import _utils as hutils
-framework_path = hutils.get_raven_loc()
-sys.path.append(framework_path)
-from utils import InputData, InputTypes
-from BaseClasses import MessageUser
 from ValuedParams import Parametric
 from ValuedParams import factory as VPFactory
 
+framework_path = hutils.get_raven_loc()
+sys.path.append(framework_path)
+from ravenframework.utils import InputData, InputTypes
+from ravenframework.BaseClasses import MessageUser
+
 class ValuedParamHandler(MessageUser):
   """
     This class enables the identification of runtime-evaluated variables

src/ValuedParams/Factory.py

@@ -2,8 +2,8 @@
 # Copyright 2020, Battelle Energy Alliance, LLC
 # ALL RIGHTS RESERVED
 
-from utils import InputData, InputTypes
-from EntityFactoryBase import EntityFactory
+from ravenframework.utils import InputData, InputTypes
+from ravenframework.EntityFactoryBase import EntityFactory
 
 from .SyntheticHistory import SyntheticHistory
 from .ROM import ROM

src/ValuedParams/ValuedParam.py

@@ -10,8 +10,8 @@
 import _utils as hutils
 framework_path = hutils.get_raven_loc()
 sys.path.append(framework_path)
-from utils import InputData, InputTypes
-from BaseClasses import MessageUser
+from ravenframework.utils import InputData, InputTypes
+from ravenframework.BaseClasses import MessageUser
 
 # class for potentially dynamically-evaluated quantities
 class ValuedParam(MessageUser):

src/_utils.py

@@ -4,11 +4,11 @@
 """
   utilities for use within heron
 """
-import os
 import sys
 import importlib
 import xml.etree.ElementTree as ET
 import warnings
+from os import path
 
 
 def get_heron_loc():
@@ -17,7 +17,7 @@ def get_heron_loc():
     @ In, None
     @ Out, loc, string, absolute location of HERON
   """
-  return os.path.abspath(os.path.join(__file__, '..', '..'))
+  return path.abspath(path.join(__file__, '..', '..'))
 
 def get_raven_loc():
   """
@@ -26,12 +26,18 @@ def get_raven_loc():
     @ In, None
     @ Out, loc, string, absolute location of RAVEN
   """
-  config = os.path.abspath(os.path.join(os.path.dirname(__file__),'..','.ravenconfig.xml'))
-  if not os.path.isfile(config):
-    raise IOError('HERON config file not found at "{}"! Has HERON been installed as a plugin in a RAVEN installation?'
-                  .format(config))
-  loc = ET.parse(config).getroot().find('FrameworkLocation').text
-  return loc
+  config = path.abspath(path.join(path.dirname(__file__),'..','.ravenconfig.xml'))
+  if not path.isfile(config):
+    raise IOError(
+        f'HERON config file not found at "{config}"! Has HERON been installed as a plugin in a RAVEN installation?'
+    )
+  loc = ET.parse(config).getroot().find('FrameworkLocation')
+  assert loc is not None and loc.text is not None
+  # The addition of ravenframework as an installable package requires
+  # adding the raven directory to the PYTHONPATH instead of adding
+  # ravenframework. We will expect '.ravenconfig.xml' to point to
+  # raven/ravenframework always, so this is why we grab the parent dir.
+  return path.abspath(path.dirname(loc.text))
 
 def get_cashflow_loc(raven_path=None):
   """
@@ -41,7 +47,7 @@ def get_cashflow_loc(raven_path=None):
   """
   if raven_path is None:
     raven_path = get_raven_loc()
-  plugin_handler_dir = os.path.join(raven_path, '..', 'scripts')
+  plugin_handler_dir = path.join(raven_path, '..', 'scripts')
   sys.path.append(plugin_handler_dir)
   plugin_handler = importlib.import_module('plugin_handler')
   sys.path.pop()
@@ -91,7 +97,7 @@ def get_synthhist_structure(fpath):
   # TODO could this be a function of the ROM itself?
   # TODO or could we interrogate the ROM directly instead of the XML?
   raven_loc = get_raven_loc()
-  scripts_path = os.path.join(raven_loc, '..', 'scripts')
+  scripts_path = path.join(raven_loc, '..', 'scripts')
   sys.path.append(scripts_path)
   from externalROMloader import ravenROMexternal as ravenROM
   # Why should we get warnings from RAVEN when we are just trying to write an input file.

src/base.py

@@ -10,7 +10,7 @@
 import _utils as hutils
 raven_path = hutils.get_raven_loc()
 sys.path.append(os.path.expanduser(raven_path))
-from BaseClasses import MessageUser
+from ravenframework.BaseClasses import MessageUser
 
 
 class Base(MessageUser):

src/dispatch/CustomDispatcher.py

@@ -8,7 +8,7 @@
 import inspect
 import numpy as np
 
-from utils import utils, InputData, InputTypes
+from ravenframework.utils import utils, InputData, InputTypes
 from .Dispatcher import Dispatcher
 from .DispatchState import NumpyState
 

src/dispatch/Dispatcher.py

@@ -4,8 +4,8 @@
 """
   Base class for dispatchers.
 """
-from utils import InputData, InputTypes
-from BaseClasses import MessageUser, InputDataUser
+from ravenframework.utils import InputData, InputTypes
+from ravenframework.BaseClasses import MessageUser, InputDataUser
 
 class Dispatcher(MessageUser, InputDataUser):
   """

src/dispatch/pyomo_dispatch.py

@@ -17,7 +17,7 @@
 import pyomo.environ as pyo
 from pyomo.opt import SolverStatus, TerminationCondition
 
-from utils import InputData, InputTypes
+from ravenframework.utils import InputData, InputTypes
 
 # allows pyomo to solve on threaded processes
 import pyutilib.subprocess.GlobalData

src/input_loader.py

@@ -15,7 +15,7 @@
 import _utils as hutils
 raven_path = hutils.get_raven_loc()
 sys.path.append(raven_path)
-from utils import xmlUtils
+from ravenframework.utils import xmlUtils
 sys.path.pop()
 
 

src/main.py

@@ -4,18 +4,15 @@
 """
   Runs HERON.
 """
-
-
 import os
 import sys
 import argparse
 import input_loader
 from base import Base
 import _utils as hutils
-raven_path = os.path.abspath(os.path.join(hutils.get_raven_loc(), '..')) #os.path.dirname(__file__)#'~/projects/raven/raven_framework'
-sys.path.append(raven_path)
-from MessageHandler import MessageHandler
-sys.path.pop()
+
+sys.path.append(hutils.get_raven_loc())
+from ravenframework.MessageHandler import MessageHandler
 
 
 class HERON(Base):

src/validators/ExampleValidator.py

@@ -6,7 +6,7 @@
 """
 import numpy as np
 
-from utils import InputData, InputTypes
+from ravenframework.utils import InputData, InputTypes
 
 from .Validator import Validator
 

src/validators/Validator.py

@@ -4,7 +4,7 @@
 """
   Base class for validators.
 """
-from utils import InputData, InputTypes
+from ravenframework.utils import InputData, InputTypes
 
 class Validator:
   """

templates/template_driver.py

@@ -21,7 +21,7 @@
 sys.path.pop()
 
 # get raven location
-RAVEN_LOC = hutils.get_raven_loc()
+RAVEN_LOC = os.path.abspath(os.path.join(hutils.get_raven_loc(), "ravenframework"))
 CF_LOC = hutils.get_cashflow_loc(raven_path=RAVEN_LOC)
 if CF_LOC is None:
   raise RuntimeError('TEAL has not been found!\n' +
@@ -33,8 +33,8 @@
 sys.path.pop()
 
 sys.path.append(os.path.join(RAVEN_LOC, '..'))
-from utils import xmlUtils
-from InputTemplates.TemplateBaseClass import Template as TemplateBase
+from ravenframework.utils import xmlUtils
+from ravenframework.InputTemplates.TemplateBaseClass import Template as TemplateBase
 sys.path.pop()
 
 class Template(TemplateBase, Base):

tests/integration_tests/mechanics/cashflows/gold/Runs_o/sweep.csv

@@ -1,3 +1,3 @@
-source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,ProbabilityWeight-source_capacity,ProbabilityWeight,prefix,PointProbability
-1.0,-2.0,3894.83025031,0.0,3894.83025031,3894.83025031,3894.83025031,3894.83025031,3894.83025031,3.0,0.0,0.5,0.5,1,1.0
-2.0,-2.0,7791.81176253,1.11389897155e-12,7791.81176253,7791.81176253,7791.81176253,7791.81176253,7791.81176253,3.0,1.24077091883e-24,0.5,0.5,2,1.0
+source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,prefix,PointProbability,ProbabilityWeight,ProbabilityWeight-source_capacity
+1.0,-2.0,5392.52224458,0.0,5392.52224458,5392.52224458,5392.52224458,5392.52224458,5392.52224458,3.0,0.0,1,1.0,0.5,0.5
+2.0,-2.0,10787.5312585,0.0,10787.5312585,10787.5312585,10787.5312585,10787.5312585,10787.5312585,3.0,0.0,2,1.0,0.5,0.5

tests/integration_tests/mechanics/labels/gold/Sweep_Runs_o/sweep.csv

@@ -1,3 +1,3 @@
-steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,state_label,strategy_label,tax_label,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,PointProbability,ProbabilityWeight-steamer_capacity,prefix,ProbabilityWeight
-1.0,-100.0,-2.0,-1e+200,Idaho,LNHR,CarbonTax,8.01709080133,4.58079852241,9.2637398142,13.289769438,-0.752129690915,-0.752129690915,13.289769438,10.0,20.9837151029,0.010101010101,0.5,1,0.5
-100.0,-100.0,-2.0,-1e+200,Idaho,LNHR,CarbonTax,403.706418539,556.60904555,531.651638685,1073.28499995,-594.408206264,-594.408206264,1073.28499995,10.0,309813.629588,0.010101010101,0.5,2,0.5
+steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,state_label,strategy_label,tax_label,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,ProbabilityWeight-steamer_capacity,prefix,ProbabilityWeight,PointProbability
+1.0,-100.0,-2.0,-1e+200,Idaho,LNHR,CarbonTax,8.01709080133,4.58079852241,9.2637398142,13.289769438,-0.752129690915,0.718499778313,13.1629332767,10.0,20.9837151029,0.5,1,0.5,0.010101010101
+100.0,-100.0,-2.0,-1e+200,Idaho,LNHR,CarbonTax,403.706418539,556.60904555,531.651638685,1073.28499995,-594.408206264,-435.425958129,1068.97222437,10.0,309813.629588,0.5,2,0.5,0.010101010101

tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv

@@ -1,6 +1,6 @@
 iteration,accepted,source_capacity,sink_capacity,mean_NPV
-0.0,first,1.05,-2.0,4089.65128702
-1.0,accepted,1.25,-2.0,4868.97075939
-2.0,accepted,1.45,-2.0,5648.34006026
-3.0,accepted,1.85,-2.0,7207.20213377
-4.0,accepted,2.0,-2.0,7791.81176253
+0.0,first,1.05,-2.0,5662.24028348
+1.0,accepted,1.25,-2.0,6741.15327388
+2.0,accepted,1.45,-2.0,7820.12386397
+3.0,accepted,1.85,-2.0,9978.20777236
+4.0,accepted,2.0,-2.0,10787.5312585

tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/sweep.csv

@@ -1,3 +1,3 @@
-source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,prefix,ProbabilityWeight-source_capacity,PointProbability,ProbabilityWeight
-1.0,-2.0,3894.83025031,0.0,3894.83025031,3894.83025031,3894.83025031,3894.83025031,3894.83025031,3.0,0.0,1,0.5,1.0,0.5
-2.0,-2.0,7791.81176253,1.11389897155e-12,7791.81176253,7791.81176253,7791.81176253,7791.81176253,7791.81176253,3.0,1.24077091883e-24,2,0.5,1.0,0.5
+source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV,max_NPV,min_NPV,perc_5_NPV,perc_95_NPV,samp_NPV,var_NPV,ProbabilityWeight-source_capacity,prefix,ProbabilityWeight,PointProbability
+1.0,-2.0,5392.52224458,0.0,5392.52224458,5392.52224458,5392.52224458,5392.52224458,5392.52224458,3.0,0.0,0.5,1,0.5,1.0
+2.0,-2.0,10787.5312585,0.0,10787.5312585,10787.5312585,10787.5312585,10787.5312585,10787.5312585,3.0,0.0,0.5,2,0.5,1.0

tests/unit_tests/ValuedParams/testParametric.py

@@ -18,10 +18,9 @@
 sys.path.pop()
 
 # Load RAVEN tools
-framework_path = hutils.get_raven_loc()
-sys.path.append(framework_path)
-from utils import InputData, xmlUtils,InputTypes
-import MessageHandler
+sys.path.append(hutils.get_raven_loc())
+from ravenframework.utils import InputData, xmlUtils,InputTypes
+import ravenframework.MessageHandler
 sys.path.pop()
 
 results = {"pass":0, "fail":0}

tests/unit_tests/testComponent.py

@@ -15,10 +15,9 @@
 sys.path.pop()
 
 # Load RAVEN tools
-framework_path = hutils.get_raven_loc()
-sys.path.append(framework_path)
-from utils import InputData, xmlUtils,InputTypes
-import MessageHandler
+sys.path.append(hutils.get_raven_loc())
+from ravenframework.utils import InputData, xmlUtils,InputTypes
+import ravenframework.MessageHandler as MessageHandler
 sys.path.pop()
 
 results = {"pass":0,"fail":0}