User specified return statistics (#197)

* revising optimization_metrics_mapping to a more general form

* add 'result_statistics' node to Cases for documentation and parameterInputFactory

* implement _read_result_statistics to fill dictionary of desired return statistics

* set up template outer.xml to receive GRO_outer_results and add GRO_outer_results to opt_soln

* modify outer.xml to accommodate user requested result statistics

* enabling user specified GRO_final_return in inner.xml

* insert PostProcessor subnodes, works for sweep mode

* implement sweep return of multiple percentiles and VaR (RAVEN work needed to enable)

* updating optimization case to allow user specified return statistics

* getting sweep tests to pass

* PostProcessor was not filling correctly for sweep mode

* handle default 'mean_NPV', 'std_NPV', 'med_NPV'

* cleaning up the code

* regold and update existing test due to additional output for opt cases

* adding test to demonstrate requesting statistics

* cleaning up user manual material

* hopefully this fixes the trailing whitespace issue

src/__init__.py

@@ -1,3 +1,3 @@
 
 # Copyright 2020, Battelle Energy Alliance, LLC
-# ALL RIGHTS RESERVED
+# ALL RIGHTS RESERVED

src/dispatch/__init__.py

@@ -1,3 +1,3 @@
 
 # Copyright 2020, Battelle Energy Alliance, LLC
-# ALL RIGHTS RESERVED
+# ALL RIGHTS RESERVED

templates/__init__.py

@@ -1,3 +1,3 @@
 
 # Copyright 2020, Battelle Energy Alliance, LLC
-# ALL RIGHTS RESERVED
+# ALL RIGHTS RESERVED

templates/inner.xml

@@ -45,10 +45,7 @@
     <Group name='GRO_armasamples_in_scalar'>scaling, GRO_capacities</Group>
     <Group name='GRO_armasamples_out_scalar'>NPV</Group>
 
-    <Group name='GRO_final_return'>
-      mean_NPV, std_NPV, med_NPV, max_NPV, min_NPV,
-      perc_5_NPV, perc_95_NPV, samp_NPV, var_NPV
-    </Group>
+    <Group name='GRO_final_return'></Group>
 
     <Group name="GRO_full_dispatch"></Group>
     <Group name="GRO_full_dispatch_indices"></Group>
@@ -103,14 +100,6 @@
     </EnsembleModel>
 
     <PostProcessor name="statistics" subType="EconomicRatio">
-      <expectedValue prefix="mean">NPV</expectedValue>
-      <sigma prefix="std">NPV</sigma>
-      <median prefix="med">NPV</median>
-      <maximum prefix="max">NPV</maximum>
-      <minimum prefix="min">NPV</minimum>
-      <percentile prefix="perc">NPV</percentile>
-      <samples prefix="samp">NPV</samples>
-      <variance prefix="var">NPV</variance>
     </PostProcessor>
   </Models>
 

templates/outer.xml

@@ -36,7 +36,7 @@
 
   <VariableGroups>
     <Group name='GRO_capacities'></Group>
-    <Group name='GRO_outer_results'>mean_NPV, std_NPV, med_NPV, max_NPV, min_NPV, perc_5_NPV, perc_95_NPV, samp_NPV, var_NPV</Group>
+    <Group name='GRO_outer_results'></Group>
     <Group name='GRO_outer_debug_dispatch'></Group>
     <Group name='GRO_outer_debug_synthetics'></Group>
   </VariableGroups>
@@ -48,11 +48,11 @@
     </PointSet>
     <PointSet name='opt_eval'>
       <Input>GRO_capacities</Input>
-      <Output>mean_NPV</Output>
+      <Output>GRO_outer_results</Output>
     </PointSet>
     <PointSet name='opt_soln'>
       <Input>trajID</Input>
-      <Output>iteration, accepted, GRO_capacities, mean_NPV</Output>
+      <Output>iteration, accepted, GRO_capacities, GRO_outer_results</Output>
     </PointSet>
   </DataObjects>
 

tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv

@@ -1,6 +1,6 @@
-iteration,accepted,source_capacity,sink_capacity,mean_NPV
-0.0,first,1.05,-2.0,5662.24028348
-1.0,accepted,1.25,-2.0,6741.15327388
-2.0,accepted,1.45,-2.0,7820.12386397
-3.0,accepted,1.85,-2.0,9978.20777236
-4.0,accepted,2.0,-2.0,10787.5312585
+iteration,accepted,source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV
+0.0,first,1.05,-2.0,5662.24028348,0.0,5662.24028348
+1.0,accepted,1.25,-2.0,6741.15327388,1.11389897155e-12,6741.15327388
+2.0,accepted,1.45,-2.0,7820.12386397,1.11389897155e-12,7820.12386397
+3.0,accepted,1.85,-2.0,9978.20777236,0.0,9978.20777236
+4.0,accepted,2.0,-2.0,10787.5312585,0.0,10787.5312585

tests/integration_tests/mechanics/multirun_sweep_opt/heron_input_opt.xml

@@ -11,7 +11,7 @@
 
     <Case name="Runs">
       <mode>opt</mode>
-      <num_arma_samples>1</num_arma_samples>
+      <num_arma_samples>3</num_arma_samples>
       <time_discretization>
         <time_variable>Time</time_variable>
         <end_time>2</end_time>

tests/integration_tests/mechanics/optimization_settings/gold/Opt_Runs_o/opt_soln_0.csv

@@ -1,9 +1,9 @@
-iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,VaR_0.05_NPV
-0.0,first,1.45,-100.0,-2.0,-2002.5,-22.7414534568
-1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,-62.6657909961
-2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,-69.0824804617
-3.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115
-4.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115
-5.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115
-6.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115
-7.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115
+iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,VaR_0.05_NPV,mean_NPV,std_NPV,med_NPV
+0.0,first,1.45,-100.0,-2.0,-2002.5,-22.7414534568,22.7414534595,2.71300300081e-09,22.7414534606
+1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,-62.6657908037,62.665790811,7.47587185876e-09,62.6657908142
+2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,-69.0824804004,69.0824804038,3.94466916627e-09,69.0824804034
+3.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256
+4.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256
+5.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256
+6.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256
+7.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256

tests/integration_tests/mechanics/result_statistics/gold/Sweep_Runs_o/sweep.csv

@@ -0,0 +1,3 @@
+BOP_capacity,Electric_Grid_capacity,mean_NPV,std_NPV,med_NPV,min_NPV,max_NPV,var_NPV,perc_5_NPV,perc_95_NPV,varCoeff_NPV,skew_NPV,kurt_NPV,samp_NPV,sharpe_NPV,sortino_median_NPV,glr_median_NPV,es_0.05_NPV,VaR_0.05_NPV,PointProbability,ProbabilityWeight-BOP_capacity,prefix,ProbabilityWeight
+1095.0,-1e+200,32365.3088338,3.28481336702e-06,32365.3088333,32365.3088309,32365.3088374,1.07899988562e-11,32365.3088311,32365.308837,1.01491797402e-10,0.714285575694,-1407482650.22,3.0,9853013007.91,22895543491.7,1.6562994969,-32365.3088309,-32365.3088311,0.333333333333,0.5,1,0.5
+1098.0,-1e+200,32453.9809128,3.29381023062e-06,32453.9809123,32453.9809098,32453.9809164,1.08491858354e-11,32453.9809101,32453.980916,1.01491716516e-10,0.714298331064,-2105487593.27,3.0,9853020860.49,22895675664.7,1.65631195711,-32453.9809098,-32453.9809101,0.333333333333,0.5,2,0.5

tests/integration_tests/mechanics/result_statistics/heron_input.xml

@@ -0,0 +1,93 @@
+<HERON>
+  <TestInfo>
+    <name>result_statistics</name>
+    <author>dgarrett622</author>
+    <created>2022-08-04</created>
+    <description>
+      Demonstrates returning desired statistics in output.
+      Based on var_demand_var_price test.
+    </description>
+    <classesTested>HERON</classesTested>
+  </TestInfo>
+
+  <Case name="Sweep_Runs">
+    <mode>sweep</mode>
+    <num_arma_samples>3</num_arma_samples>
+    <time_discretization>
+      <time_variable>Time</time_variable>
+      <end_time>2</end_time>
+      <num_steps>21</num_steps>
+    </time_discretization>
+    <economics>
+      <ProjectTime>3</ProjectTime>
+      <DiscountRate>0.08</DiscountRate>
+      <tax>0.0</tax>
+      <inflation>0.0</inflation>
+      <verbosity>50</verbosity>
+    </economics>
+    <dispatcher>
+      <pyomo/>
+    </dispatcher>
+    <result_statistics>
+      <minimum/>
+      <maximum/>
+      <variance/>
+      <percentile/>
+      <variationCoefficient/>
+      <skewness/>
+      <kurtosis/>
+      <samples/>
+      <sharpeRatio/>
+      <sortinoRatio threshold="median"/>
+      <gainLossRatio threshold="median"/>
+      <expectedShortfall/>
+      <valueAtRisk/>
+    </result_statistics>
+  </Case>
+
+  <Components>
+    <Component name="BOP">
+      <produces resource="electricity" dispatch="fixed">
+        <capacity resource="electricity">
+          <sweep_values>1095,1098</sweep_values>
+        </capacity>
+      </produces>
+      <economics>
+        <lifetime>27</lifetime>
+      </economics>
+    </Component>
+
+    <Component name="Electric_Grid">
+      <demands resource="electricity" dispatch="independent">
+        <capacity>
+          <fixed_value>-1e200</fixed_value>
+        </capacity>
+      </demands>
+      <economics>
+        <lifetime>3</lifetime>
+        <CashFlow name="e_sales" type="repeating" taxable='True' inflation='none' mult_target='False'>
+          <driver>
+            <activity>electricity</activity>
+            <multiplier>-1</multiplier>
+          </driver>
+          <reference_price>
+            <ARMA variable="Signal">Speed</ARMA>
+          </reference_price>
+          <reference_driver>
+            <fixed_value>1</fixed_value>
+          </reference_driver>
+          <scaling_factor_x>
+            <fixed_value>1</fixed_value>
+          </scaling_factor_x>
+        </CashFlow>
+      </economics>
+    </Component>
+
+  </Components>
+
+  <DataGenerators>
+    <ARMA name='Speed' variable="Signal">%HERON%/tests/integration_tests/ARMA/Sine/arma.pk</ARMA>
+    <Function name="transfers">transfers.py</Function>
+  </DataGenerators>
+
+</HERON>

tests/integration_tests/mechanics/result_statistics/tests

@@ -0,0 +1,14 @@
+[Tests]
+ [./ResultStatistics]
+  type = HeronIntegration
+  input = heron_input.xml
+  # prereq = SineArma
+  [./csv]
+    type = UnorderedCSV
+    output = 'Sweep_Runs_o/sweep.csv'
+    zero_threshold = 1e-6
+    rel_err = 1e-5
+  [../]
+ [../]
+
+[]

tests/integration_tests/mechanics/result_statistics/transfers.py

@@ -0,0 +1,23 @@
+
+# Copyright 2020, Battelle Energy Alliance, LLC
+# ALL RIGHTS RESERVED
+"""
+  Implements transfer functions
+"""
+
+def power_conversion(data, meta):
+  """
+    How to get power from the incoming signal
+  """
+  # get the signal (year, time) from RAVEN ARMA
+  ## NOTE this behaves completely different if you remove
+  # the 1.0, and I have no idea why. Leave it there, and
+  # you get the correct analytic results.
+  signal = 1.0 * meta['raven_vars']['Signal'][:, :]
+  # what time step are we currently at?
+  index = meta['HERON']['time_index']
+  # return the entry from the appropriate index
+  power = signal[index, 0] + 10
+  # set the value to return
+  data['electricity'] = power
+  return(data, meta)

tests/integration_tests/workflows/production_flex_opt/gold/Opt_Runs_o/opt_soln_0.csv

@@ -1,7 +1,5 @@
-iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,mean_NPV
-0.0,first,1.45,-100.0,-2.0,-2002.5,22.7414534595
-1.0,accepted,2.08639610307,-100.0,-2.0,-2002.5,32.7225393657
-2.0,accepted,2.72279220614,-100.0,-2.0,-2002.5,42.7036221353
-3.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,62.665790811
-4.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,69.0824804038
-5.0,accepted,10.0,-100.0,-2.0,-2002.5,77.7220918232
+iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,mean_NPV,std_NPV,med_NPV
+0.0,first,1.45,-100.0,-2.0,-2002.5,22.7414534595,2.71300300081e-09,22.7414534606
+1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,62.665790811,7.47587185876e-09,62.6657908142
+2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,69.0824804038,3.94466916627e-09,69.0824804034
+3.0,accepted,10.0,-100.0,-2.0,-2002.5,77.7220918232,1.21349097015e-08,77.7220918256