From 7d159f925eeca0ff93964ca08f921cf2ff1c9b95 Mon Sep 17 00:00:00 2001 From: Daniel Garrett Date: Thu, 4 Aug 2022 14:35:17 -0600 Subject: [PATCH] User specified return statistics (#197) * revising optimization_metrics_mapping to a more general form * add 'result_statistics' node to Cases for documentation and parameterInputFactory * implement _read_result_statistics to fill dictionary of desired return statistics * set up template outer.xml to receive GRO_outer_results and add GRO_outer_results to opt_soln * modify outer.xml to accommodate user requested result statistics * enabling user specified GRO_final_return in inner.xml * insert PostProcessor subnodes, works for sweep mode * implement sweep return of multiple percentiles and VaR (RAVEN work needed to enable) * updating optimization case to allow user specified return statistics * getting sweep tests to pass * PostProcessor was not filling correctly for sweep mode * handle default 'mean_NPV', 'std_NPV', 'med_NPV' * cleaning up the code * regold and update existing test due to additional output for opt cases * adding test to demonstrate requesting statistics * cleaning up user manual material * hopefully this fixes the trailing whitespace issue --- src/Cases.py | 210 +++++++++++----- src/__init__.py | 2 +- src/dispatch/__init__.py | 2 +- templates/__init__.py | 2 +- templates/inner.xml | 13 +- templates/outer.xml | 6 +- templates/template_driver.py | 234 ++++++++++++++---- .../gold/Runs_o/opt_soln_0.csv | 12 +- .../multirun_sweep_opt/heron_input_opt.xml | 2 +- .../gold/Opt_Runs_o/opt_soln_0.csv | 18 +- .../gold/Sweep_Runs_o/sweep.csv | 3 + .../result_statistics/heron_input.xml | 93 +++++++ .../mechanics/result_statistics/tests | 14 ++ .../mechanics/result_statistics/transfers.py | 23 ++ .../gold/Opt_Runs_o/opt_soln_0.csv | 12 +- 15 files changed, 497 insertions(+), 149 deletions(-) create mode 100644 tests/integration_tests/mechanics/result_statistics/gold/Sweep_Runs_o/sweep.csv create mode 100644 tests/integration_tests/mechanics/result_statistics/heron_input.xml create mode 100644 tests/integration_tests/mechanics/result_statistics/tests create mode 100644 tests/integration_tests/mechanics/result_statistics/transfers.py diff --git a/src/Cases.py b/src/Cases.py index 2f4fd918..b67ea693 100644 --- a/src/Cases.py +++ b/src/Cases.py @@ -7,14 +7,11 @@ from __future__ import unicode_literals, print_function import os import sys -import copy import importlib import numpy as np from HERON.src.base import Base -from HERON.src import Components -from HERON.src import Placeholders from HERON.src.dispatch.Factory import known as known_dispatchers from HERON.src.dispatch.Factory import get_class as get_dispatcher @@ -28,7 +25,7 @@ import HERON.src._utils as hutils framework_path = hutils.get_raven_loc() sys.path.append(framework_path) -from ravenframework.utils import InputData, InputTypes, xmlUtils +from ravenframework.utils import InputData, InputTypes class Case(Base): """ @@ -36,23 +33,28 @@ class Case(Base): TODO this case is for "sweep-opt", need to make a superclass for generic """ - # metrics that can be used for objective in optimization mapped from RAVEN name to result name (prefix) - # 'default' is the default type of optimization (min/max) - optimization_metrics_mapping = {'expectedValue': {'prefix': 'mean', 'default': 'max'}, - 'minimum': {'prefix': 'min', 'default': 'max'}, - 'maximum': {'prefix': 'max', 'default': 'max'}, - 'median': {'prefix': 'med', 'default': 'max'}, - 'variance': {'prefix': 'var', 'default': 'min'}, - 'sigma': {'prefix': 'std', 'default': 'min'}, - 'percentile': {'prefix': 'perc', 'default': 'max'}, - 'variationCoefficient': {'prefix': 'varCoeff', 'default': 'min'}, - 'skewness': {'prefix': 'skew', 'default': 'min'}, - 'kurtosis': {'prefix': 'kurt', 'default': 'min'}, - 'sharpeRatio': {'prefix': 'sharpe', 'default': 'max'}, - 'sortinoRatio': {'prefix': 'sortino', 'default': 'max'}, - 'gainLossRatio': {'prefix': 'glr', 'default': 'max'}, - 'expectedShortfall': {'prefix': 'es', 'default': 'min'}, - 'valueAtRisk': {'prefix': 'VaR', 'default': 'min'}} + # metrics that can be used for objective in optimization or returned with results + # each metric contains a dictionary with the following keys: + # 'prefix' - printed result name + # 'optimization_default' - 'min' or 'max' for optimization + # 'percent' (only for percentile) - list of percentiles to return + # 'threshold' (only for sortinoRatio, gainLossRatio, expectedShortfall, valueAtRisk) - threshold value for calculation + metrics_mapping = {'expectedValue': {'prefix': 'mean', 'optimization_default': 'max'}, + 'minimum': {'prefix': 'min', 'optimization_default': 'max'}, + 'maximum': {'prefix': 'max', 'optimization_default': 'max'}, + 'median': {'prefix': 'med', 'optimization_default': 'max'}, + 'variance': {'prefix': 'var', 'optimization_default': 'min'}, + 'sigma': {'prefix': 'std', 'optimization_default': 'min'}, + 'percentile': {'prefix': 'perc', 'optimization_default': 'max', 'percent': ['5', '95']}, + 'variationCoefficient': {'prefix': 'varCoeff', 'optimization_default': 'min'}, + 'skewness': {'prefix': 'skew', 'optimization_default': 'min'}, + 'kurtosis': {'prefix': 'kurt', 'optimization_default': 'min'}, + 'samples': {'prefix': 'samp'}, + 'sharpeRatio': {'prefix': 'sharpe', 'optimization_default': 'max'}, + 'sortinoRatio': {'prefix': 'sortino', 'optimization_default': 'max', 'threshold': 'median'}, + 'gainLossRatio': {'prefix': 'glr', 'optimization_default': 'max', 'threshold': 'median'}, + 'expectedShortfall': {'prefix': 'es', 'optimization_default': 'min', 'threshold': ['0.05']}, + 'valueAtRisk': {'prefix': 'VaR', 'optimization_default': 'min', 'threshold': ['0.05']}} #### INITIALIZATION #### @classmethod @@ -226,9 +228,9 @@ def get_input_specs(cls): # optimization settings optimizer = InputData.parameterInputFactory('optimization_settings', - descr=r"""node that defines the settings to be used for the optimizer in + descr=r"""This node defines the settings to be used for the optimizer in the ``outer'' run.""") - metric_options = InputTypes.makeEnumType('MetricOptions', 'MetricOptionsType', list(cls.optimization_metrics_mapping.keys())) + metric_options = InputTypes.makeEnumType('MetricOptions', 'MetricOptionsType', list(cls.metrics_mapping.keys())) desc_metric_options = r"""determines the statistical metric (calculated by RAVEN BasicStatistics or EconomicRatio PostProcessors) from the ``inner'' run to be used as the objective in the ``outer'' optimization. @@ -316,6 +318,33 @@ def get_input_specs(cls): dispatch_vars.addSub(value_param) input_specs.addSub(dispatch_vars) + # result statistics + result_stats = InputData.parameterInputFactory('result_statistics', + descr=r"""This node defines the additional statistics + to be returned with the results. The statistics + \texttt{expectedValue} (prefix ``mean''), + \texttt{sigma} (prefix ``std''), and \texttt{median} + (prefix ``med'') are always returned with the results. + Each subnode is the RAVEN-style name of the desired + return statistic.""") + for stat in cls.metrics_mapping: + if stat not in ['expectedValue', 'sigma', 'median']: + statistic = InputData.parameterInputFactory(stat, strictMode=True, + descr=r"""{} uses the prefix ``{}'' in the result output.""".format(stat, cls.metrics_mapping[stat]['prefix'])) + if stat == 'percentile': + statistic.addParam('percent', param_type=InputTypes.StringType, + descr=r"""requested percentile (a floating point value between 0.0 and 100.0). + When no percent is given, returns both 5th and 95th percentiles.""") + elif stat in ['sortinoRatio', 'gainLossRatio']: + statistic.addParam('threshold', param_type=InputTypes.StringType, + descr=r"""requested threshold (``median" or ``zero").""", default='``median"') + elif stat in ['expectedShortfall', 'valueAtRisk']: + statistic.addParam('threshold', param_type=InputTypes.StringType, + descr=r"""requested threshold (a floating point value between 0.0 and 1.0).""", + default='``0.05"') + result_stats.addSub(statistic) + input_specs.addSub(result_stats) + return input_specs def __init__(self, run_dir, **kwargs): @@ -325,45 +354,49 @@ def __init__(self, run_dir, **kwargs): @ Out, None """ Base.__init__(self, **kwargs) - self.name = None # case name - self._mode = None # extrema to find: opt, sweep - self._metric = 'NPV' # UNUSED (future work); economic metric to focus on: lcoe, profit, cost - self.run_dir = run_dir # location of HERON input file - self._verbosity = 'all' # default verbosity for RAVEN inner/outer - - self.dispatch_name = None # name of dispatcher to use - self.dispatcher = None # type of dispatcher to use - self.validator_name = None # name of dispatch validation to use - self.validator = None # type of dispatch validation to use - self.dispatch_vars = {} # non-component optimization ValuedParams - - self.outerParallel = 0 # number of outer parallel runs to use - self.innerParallel = 0 # number of inner parallel runs to use - - self._diff_study = None # is this only a differential study? - self._num_samples = 1 # number of ARMA stochastic samples to use ("denoises") - self._hist_interval = None # time step interval, time between production points - self._hist_len = None # total history length, in same units as _hist_interval - self._num_hist = None # number of history steps, hist_len / hist_interval - self._global_econ = {} # global economics settings, as a pass-through - self._increments = {} # stepwise increments for resource balancing - self._time_varname = 'time' # name of the time-variable throughout simulation - self._year_varname = 'Year' # name of the year-variable throughout simulation - self._labels = {} # extra information pertaining to current case - self.debug = { # debug options, as enabled by the user (defaults included) - 'enabled': False, # whether to enable debug mode - 'inner_samples': 1, # how many inner realizations to sample - 'macro_steps': 1, # how many "years" for inner realizations - 'dispatch_plot': True # whether to output a plot in debug mode + self.name = None # case name + self._mode = None # extrema to find: opt, sweep + self._metric = 'NPV' # TODO: future work - economic metric to focus on: lcoe, profit, cost + self.run_dir = run_dir # location of HERON input file + self._verbosity = 'all' # default verbosity for RAVEN inner/outer + + self.dispatch_name = None # name of dispatcher to use + self.dispatcher = None # type of dispatcher to use + self.validator_name = None # name of dispatch validation to use + self.validator = None # type of dispatch validation to use + self.dispatch_vars = {} # non-component optimization ValuedParams + + self.outerParallel = 0 # number of outer parallel runs to use + self.innerParallel = 0 # number of inner parallel runs to use + + self._diff_study = None # is this only a differential study? + self._num_samples = 1 # number of ARMA stochastic samples to use ("denoises") + self._hist_interval = None # time step interval, time between production points + self._hist_len = None # total history length, in same units as _hist_interval + self._num_hist = None # number of history steps, hist_len / hist_interval + self._global_econ = {} # global economics settings, as a pass-through + self._increments = {} # stepwise increments for resource balancing + self._time_varname = 'time' # name of the time-variable throughout simulation + self._year_varname = 'Year' # name of the year-variable throughout simulation + self._labels = {} # extra information pertaining to current case + self.debug = { # debug options, as enabled by the user (defaults included) + 'enabled': False, # whether to enable debug mode + 'inner_samples': 1, # how many inner realizations to sample + 'macro_steps': 1, # how many "years" for inner realizations + 'dispatch_plot': True # whether to output a plot in debug mode } - self.data_handling = { # data handling options - 'inner_to_outer': 'netcdf', # how to pass inner data to outer (csv, netcdf) + self.data_handling = { # data handling options + 'inner_to_outer': 'netcdf', # how to pass inner data to outer (csv, netcdf) } - self._time_discretization = None # (start, end, number) for constructing time discretization, same as argument to np.linspace - self._Resample_T = None # user-set increments for resources + self._time_discretization = None # (start, end, number) for constructing time discretization, same as argument to np.linspace + self._Resample_T = None # user-set increments for resources self._optimization_settings = None # optimization settings dictionary for outer optimization loop + self._result_statistics = { # desired result statistics (keys) dictionary with attributes (values) + 'sigma': None, # user can specify additional result statistics + 'expectedValue': None, + 'median': None} # clean up location self.run_dir = os.path.abspath(os.path.expanduser(self.run_dir)) @@ -432,6 +465,9 @@ def read_input(self, xml): self.dispatch_vars[var_name] = vp elif item.getName() == 'data_handling': self.data_handling = self._read_data_handling(item) + elif item.getName() == 'result_statistics': + new_result_statistics = self._read_result_statistics(item) + self._result_statistics.update(new_result_statistics) # checks if self._mode is None: @@ -453,7 +489,7 @@ def read_input(self, xml): self.dispatcher.set_time_discr(self._time_discretization) self.dispatcher.set_validator(self.validator) - self.raiseADebug('Successfully initialized Case {}.'.format(self.name)) + self.raiseADebug(f'Successfully initialized Case {self.name}.') def _read_data_handling(self, node): """ @@ -558,6 +594,54 @@ def _read_optimization_settings(self, node): return opt_settings + def _read_result_statistics(self, node): + """ + Reads result statistics node + @ In, node, InputParams.ParameterInput, result statistics head node + @ Out, result_statistics, dict, result statistics settings as dictionary + """ + # result_statistics keys are statistic name value is percent, threshold value, or None + result_statistics = {} + for sub in node.subparts: + sub_name = sub.getName() + if sub_name == 'percentile': + try: + percent = sub.parameterValues['percent'] + # if multiple percents are given, set as a list + if sub_name in result_statistics: + if isinstance(result_statistics[sub_name], list): + if percent not in result_statistics[sub_name]: + result_statistics[sub_name].append(percent) + else: + result_statistics[sub_name] = [result_statistics[sub_name], percent] + else: + result_statistics[sub_name] = percent + except KeyError: + result_statistics[sub_name] = self.metrics_mapping[sub_name]['percent'] + elif sub_name in ['sortinoRatio', 'gainLossRatio']: + try: + result_statistics[sub_name] = sub.parameterValues['threshold'] + except KeyError: + result_statistics[sub_name] = self.metrics_mapping[sub_name]['threshold'] + elif sub_name in ['expectedShortfall', 'valueAtRisk']: + try: + threshold = sub.parameterValues['threshold'] + # if multiple thresholds are given, set as a list + if sub_name in result_statistics: + if isinstance(result_statistics[sub_name], list): + if threshold not in result_statistics[sub_name]: + result_statistics[sub_name].append(threshold) + else: + result_statistics[sub_name] = [result_statistics[sub_name], threshold] + else: + result_statistics[sub_name] = sub.parameterValues['threshold'] + except KeyError: + result_statistics[sub_name] = self.metrics_mapping[sub_name]['threshold'] + else: + result_statistics[sub_name] = None + + return result_statistics + def initialize(self, components, sources): """ Called after all objects are created, allows post-input initialization @@ -579,7 +663,7 @@ def __repr__(self): """ return '' - def print_me(self, tabs=0, tab=' '): + def print_me(self, tabs=0, tab=' ', **kwargs): """ Prints info about self @ In, tabs, int, number of tabs to insert before print @@ -613,8 +697,8 @@ def get_working_dir(self, which): elif which == 'inner': io = 'i' else: - raise NotImplementedError('Unrecognized working dir request: "{}"'.format(which)) - return '{case}_{io}'.format(case=self.name, io=io) + raise NotImplementedError(f'Unrecognized working dir request: "{which}"') + return f'{self.name}_{io}' def load_econ(self, components): """ @@ -631,7 +715,7 @@ def load_econ(self, components): comp_name = comp.name for cf in comp.get_cashflows(): cf_name = cf.name - indic['active'].append('{}|{}'.format(comp_name, cf_name)) + indic['active'].append(f'{comp_name}|{cf_name}') self._global_econ['Indicator'] = indic def get_econ(self, components): @@ -770,7 +854,7 @@ def _load_template(self): template_name = 'template_driver' # import template module sys.path.append(heron_dir) - module = importlib.import_module('templates.{}'.format(template_name), package="HERON") + module = importlib.import_module(f'templates.{template_name}', package="HERON") # load template, perform actions template_class = module.Template(messageHandler=self.messageHandler) template_class.loadTemplate(template_dir) diff --git a/src/__init__.py b/src/__init__.py index 1c5f9417..5e3ed630 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,3 +1,3 @@ # Copyright 2020, Battelle Energy Alliance, LLC -# ALL RIGHTS RESERVED \ No newline at end of file +# ALL RIGHTS RESERVED diff --git a/src/dispatch/__init__.py b/src/dispatch/__init__.py index 1c5f9417..5e3ed630 100644 --- a/src/dispatch/__init__.py +++ b/src/dispatch/__init__.py @@ -1,3 +1,3 @@ # Copyright 2020, Battelle Energy Alliance, LLC -# ALL RIGHTS RESERVED \ No newline at end of file +# ALL RIGHTS RESERVED diff --git a/templates/__init__.py b/templates/__init__.py index 1c5f9417..5e3ed630 100644 --- a/templates/__init__.py +++ b/templates/__init__.py @@ -1,3 +1,3 @@ # Copyright 2020, Battelle Energy Alliance, LLC -# ALL RIGHTS RESERVED \ No newline at end of file +# ALL RIGHTS RESERVED diff --git a/templates/inner.xml b/templates/inner.xml index 76bc2534..e5cb94a5 100644 --- a/templates/inner.xml +++ b/templates/inner.xml @@ -45,10 +45,7 @@ scaling, GRO_capacities NPV - - mean_NPV, std_NPV, med_NPV, max_NPV, min_NPV, - perc_5_NPV, perc_95_NPV, samp_NPV, var_NPV - + @@ -103,14 +100,6 @@ - NPV - NPV - NPV - NPV - NPV - NPV - NPV - NPV diff --git a/templates/outer.xml b/templates/outer.xml index c7895968..e0bbabbf 100644 --- a/templates/outer.xml +++ b/templates/outer.xml @@ -36,7 +36,7 @@ - mean_NPV, std_NPV, med_NPV, max_NPV, min_NPV, perc_5_NPV, perc_95_NPV, samp_NPV, var_NPV + @@ -48,11 +48,11 @@ GRO_capacities - mean_NPV + GRO_outer_results trajID - iteration, accepted, GRO_capacities, mean_NPV + iteration, accepted, GRO_capacities, GRO_outer_results diff --git a/templates/template_driver.py b/templates/template_driver.py index 88fa6577..61ae14a1 100644 --- a/templates/template_driver.py +++ b/templates/template_driver.py @@ -222,7 +222,7 @@ def _modify_outer_mode(self, template, case, components, sources): """ if case.get_mode() == 'sweep' or case.debug['enabled']: template.remove(template.find('Optimizers')) - elif case._mode == 'opt': + elif case.get_mode() == 'opt': template.remove(template.find('Samplers')) def _modify_outer_runinfo(self, template, case): @@ -278,11 +278,32 @@ def _modify_outer_vargroups(self, template, case, components, sources): caps.text = ', '.join(var_list) # outer results - if case._optimization_settings is not None: - group_outer_results = var_groups.find(".//Group[@name='GRO_outer_results']") + group_outer_results = var_groups.find(".//Group[@name='GRO_outer_results']") + # add required defaults + default_stats = [f'mean_{case._metric}', f'std_{case._metric}', f'med_{case._metric}'] + for stat in default_stats: + self._updateCommaSeperatedList(group_outer_results, stat) + # make sure user provided statistics beyond defaults get there + if any(stat not in ['expectedValue', 'sigma', 'median'] for stat in case._result_statistics): + stats_list = self._build_result_statistic_names(case) + for stat_name in stats_list: + if stat_name not in default_stats: + self._updateCommaSeperatedList(group_outer_results, stat_name) + # sweep mode has default variable names + elif case.get_mode() == 'sweep': + sweep_default = [f'mean_{case._metric}', f'std_{case._metric}', f'med_{case._metric}', f'max_{case._metric}', + f'min_{case._metric}', f'perc_5_{case._metric}', f'perc_95_{case._metric}', + f'samp_{case._metric}', f'var_{case._metric}'] + for sweep_name in sweep_default: + if sweep_name not in default_stats: + self._updateCommaSeperatedList(group_outer_results, sweep_name) + # opt mode adds optimization variable if not already there + if (case.get_mode() == 'opt') and (case._optimization_settings is not None): new_metric_outer_results = self._build_opt_metric_out_name(case) if (new_metric_outer_results != 'missing') and (new_metric_outer_results not in group_outer_results.text): + # additional results statistics have been requested, add this metric if not already present self._updateCommaSeperatedList(group_outer_results, new_metric_outer_results, position=0) + # labels group if case.get_labels(): case_labels = ET.SubElement(var_groups, 'Group', attrib={'name': 'GRO_case_labels'}) @@ -341,18 +362,6 @@ def _modify_outer_dataobjects(self, template, case, components): self._remove_by_name(DOs, ['opt_eval', 'opt_soln']) elif case.get_mode() == 'opt': self._remove_by_name(DOs, ['grid']) - # update optimization settings if provided - if (case.get_mode() == 'opt') and (case._optimization_settings is not None) and (not case.debug['enabled']): # TODO there should be a better way to handle the debug case - new_opt_objective = self._build_opt_metric_out_name(case) - # check if the metric in 'opt_eval' needs to be changed - opt_eval_output_node = DOs.find(".//PointSet[@name='opt_eval']").find('Output') - if (new_opt_objective != 'missing') and (new_opt_objective != opt_eval_output_node.text): - opt_eval_output_node.text = new_opt_objective - # check if the metric in 'opt_soln' needs to be changed - opt_soln_output = DOs.find(".//PointSet[@name='opt_soln']").find('Output') - if (new_opt_objective != 'missing') and (new_opt_objective not in opt_soln_output.text): - # remove mean_NPV and replace with new_opt_objective - opt_soln_output.text = opt_soln_output.text.replace('mean_NPV', new_opt_objective) # debug mode if case.debug['enabled']: # add debug dispatch output dataset @@ -414,7 +423,7 @@ def _modify_outer_models(self, template, case, components): text = 'Samplers|MonteCarlo@name:mc_arma_dispatch|constant@name:{}_capacity' for component in components: name = component.name - attribs = {'variable':'{}_capacity'.format(name), 'type':'input'} + attribs = {'variable': f'{name}_capacity', 'type':'input'} new = xmlUtils.newNode('alias', text=text.format(name), attrib=attribs) raven.append(new) @@ -463,7 +472,7 @@ def _modify_outer_outstreams(self, template, case, components, sources): new_opt_objective = self._build_opt_metric_out_name(case) opt_path_plot_vars = OSs.find(".//Plot[@name='opt_path']").find('vars') if (new_opt_objective != 'missing') and (new_opt_objective not in opt_path_plot_vars.text): - opt_path_plot_vars.text = opt_path_plot_vars.text.replace('mean_NPV', new_opt_objective) + opt_path_plot_vars.text = opt_path_plot_vars.text.replace(f'mean_{case._metric}', new_opt_objective) # debug mode if case.debug['enabled']: # modify normal metric output @@ -586,7 +595,7 @@ def _modify_outer_optimizers(self, template, case): except KeyError: # type was not provided, so use the default value metric_raven_name = case._optimization_settings['metric']['name'] - type_node.text = case.optimization_metrics_mapping[metric_raven_name]['default'] + type_node.text = case.metrics_mapping[metric_raven_name]['optimization_default'] # swap out convergence values (only persistence implemented now) convergence = opt_node.find('convergence') @@ -711,6 +720,7 @@ def _modify_inner(self, template, case, components, sources): self._modify_inner_components(template, case, components) self._modify_inner_caselabels(template, case) self._modify_inner_time_vars(template, case) + self._modify_inner_result_statistics(template, case) self._modify_inner_optimization_settings(template, case) self._modify_inner_data_handling(template, case) if case.debug['enabled']: @@ -845,7 +855,7 @@ def _modify_inner_components(self, template, case, components): groups = {} var_groups = template.find('VariableGroups') for tag in ['capacities', 'init_disp', 'full_dispatch']: - groups[tag] = var_groups.find(".//Group[@name='GRO_{}']".format(tag)) + groups[tag] = var_groups.find(f".//Group[@name='GRO_{tag}']") # change inner input due to components requested for component in components: @@ -876,7 +886,7 @@ def _modify_inner_components(self, template, case, components): # OR capacity is limited by a function, and we also can't handle it here, but in the dispatch. pass else: - raise NotImplementedError('Capacity from "{}" not implemented yet. Component: {}'.format(capacity, cap_name)) + raise NotImplementedError(f'Capacity from "{capacity}" not implemented yet. Component: {cap_name}') for tracker in component.get_tracking_vars(): for resource in component.get_resources(): @@ -930,27 +940,32 @@ def _modify_inner_optimization_settings(self, template, case): new_objective = self._build_opt_metric_out_name(case) # add optimization objective name to VariableGroups 'GRO_final_return' if not already there group = template.find('VariableGroups').find(".//Group[@name='GRO_final_return']") - if (new_objective != 'missing') and (new_objective not in group.text): + if group.text is None: + self._updateCommaSeperatedList(group, new_objective, postion=0) + elif (new_objective != 'missing') and (new_objective not in group.text): self._updateCommaSeperatedList(group, new_objective, position=0) # add optimization objective to PostProcessor list if not already there + pp_node = template.find('Models').find(".//PostProcessor[@name='statistics']") if new_objective != 'missing': - pp_node = template.find('Models').find(".//PostProcessor[@name='statistics']") raven_metric_name = case._optimization_settings['metric']['name'] - prefix = case.optimization_metrics_mapping[raven_metric_name]['prefix'] + prefix = case.metrics_mapping[raven_metric_name]['prefix'] if pp_node.find(raven_metric_name) is None: # add subnode to PostProcessor - if 'threshold' in case._optimization_settings['metric'].keys(): + if 'threshold' in case._optimization_settings['metric']: if raven_metric_name in ['valueAtRisk', 'expectedShortfall']: threshold = str(case._optimization_settings['metric']['threshold']) else: threshold = case._optimization_settings['metric']['threshold'] - # TODO should NPV be the only metric available? - new_node = xmlUtils.newNode(raven_metric_name, text='NPV', + new_node = xmlUtils.newNode(raven_metric_name, text=case._metric, attrib={'prefix': prefix, 'threshold': threshold}) + elif 'percent' in case._optimization_settings['metric']: + percent = str(case._optimization_settings['metric']['percent']) + new_node = xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix, + 'percent': percent}) else: - # TODO should NPV be the only metric available? - new_node = xmlUtils.newNode(raven_metric_name, text='NPV', + new_node = xmlUtils.newNode(raven_metric_name, text=case._metric, attrib={'prefix': prefix}) pp_node.append(new_node) else: @@ -960,11 +975,120 @@ def _modify_inner_optimization_settings(self, template, case): if prefix != subnode.attrib['prefix']: subnode.attrib['prefix'] = prefix # percentile has additional parameter to check - if 'percent' in case._optimization_settings['metric'].keys(): - # defaults to 5 or 95 percentile - if str(int(case._optimization_settings['metric']['percent'])) not in ['5', '95']: - # update attribute - subnode.attrib['percent'] = str(case._optimization_settings['metric']['percent']) + if 'percent' in case._optimization_settings['metric']: + # see if percentile already has what we need + if str(int(case._optimization_settings['metric']['percent'])) not in subnode.attrib['percent']: + # nope, need to add the percent to the existing attribute + subnode.attrib['percent'] += ','+str(case._optimization_settings['metric']['percent']) + if 'threshold' in case._optimization_settings['metric']: + # see if the threshold is already there + if str(case._optimization_settings['metric']['threshold']) not in subnode.attrib['threshold']: + # nope, need to add the threshold to existing attribute + subnode.attrib['threshold'] += ','+str(case._optimization_settings['metric']['threshold']) + else: + # new_objective is missing, use mean_metric + if pp_node.find('expectedValue') is None: + pp_node.append(xmlUtils.newNode('expectedValue', text=case._metric, + attrib={'prefix': 'mean'})) + else: + # check that the subnode has the correct values + subnode = pp_node.find('expectedValue') + if 'mean' != subnode.attrib['prefix']: + subnode.attrib['prefix'] = 'mean' + # if no optimization settings specified, make sure mean_metric is in PostProcessor node + elif case.get_mode() == 'opt': + pp_node = template.find('Models').find(".//PostProcessor[@name='statistics']") + if pp_node.find('expectedValue') is None: + pp_node.append(xmlUtils.newNode('expectedValue', text=case._metric, + attrib={'prefix': 'mean'})) + else: + # check that the subnode has the correct values + subnode = pp_node.find('expectedValue') + if 'mean' != subnode.attrib['prefix']: + subnode.attrib['prefix'] = 'mean' + + def _modify_inner_result_statistics(self, template, case): + """ + Modifies template to include result statistics + @ In, template, xml.etree.ElementTree.Element, root of XML to modify + @ In, case, HERON Case, defining Case instance + @ Out, None + """ + # handle VariableGroups + var_groups = template.find('VariableGroups') + # final return variable group (sent to outer) + group_final_return = var_groups.find(".//Group[@name='GRO_final_return']") + # add required defaults + default_stats = [f'mean_{case._metric}', f'std_{case._metric}', f'med_{case._metric}'] + for stat in default_stats: + self._updateCommaSeperatedList(group_final_return, stat) + # make sure user provided statistics beyond defaults get there + if any(stat not in ['expectedValue', 'sigma', 'median'] for stat in case._result_statistics): + stats_list = self._build_result_statistic_names(case) + for stat_name in stats_list: + if stat_name not in default_stats: + self._updateCommaSeperatedList(group_final_return, stat_name) + # sweep mode has default variable names + elif case.get_mode() == 'sweep': + sweep_default = [f'mean_{case._metric}', f'std_{case._metric}', f'med_{case._metric}', + f'max_{case._metric}', f'min_{case._metric}', f'perc_5_{case._metric}', + f'perc_95_{case._metric}', f'samp_{case._metric}', f'var_{case._metric}'] + for sweep_name in sweep_default: + if sweep_name not in default_stats: + self._updateCommaSeperatedList(group_final_return, sweep_name) + # opt mode uses optimization variable if no other stats are given, this is handled below + if (case.get_mode == 'opt') and (case._optimization_settings is not None): + new_metric_opt_results = self._build_opt_metric_out_name(case) + if (new_metric_opt_results != 'missing') and (new_metric_opt_results not in group_final_return.text): + # additional results statistics have been requested, add this metric if not already present + self._updateCommaSeperatedList(group_final_return, new_metric_opt_results, position=0) + + # fill out PostProcessor nodes + pp_node = template.find('Models').find(".//PostProcessor[@name='statistics']") + # add default statistics + stats = ['expectedValue', 'sigma', 'median'] + prefixes = ['mean', 'std', 'med'] + for stat, pref in zip(stats, prefixes): + pp_node.append(xmlUtils.newNode(stat, text=case._metric, attrib={'prefix': pref})) + # add any user supplied statistics beyond defaults + if any(stat not in ['expectedValue', 'sigma', 'median'] for stat in case._result_statistics): + for raven_metric_name in case._result_statistics: + if raven_metric_name not in stats: + prefix = case.metrics_mapping[raven_metric_name]['prefix'] + # add subnode to PostProcessor + if raven_metric_name == 'percentile': + # add percent attribute + percent = case._result_statistics[raven_metric_name] + if isinstance(percent, list): + for p in percent: + pp_node.append(xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix, + 'percent': p})) + else: + pp_node.append(xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix, + 'percent': percent})) + elif raven_metric_name in ['valueAtRisk', 'expectedShortfall', 'sortinoRatio', 'gainLossRatio']: + threshold = case._result_statistics[raven_metric_name] + if isinstance(threshold, list): + for t in threshold: + pp_node.append(xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix, + 'threshold': t})) + else: + pp_node.append(xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix, + 'threshold': threshold})) + else: + pp_node.append(xmlUtils.newNode(raven_metric_name, text=case._metric, + attrib={'prefix': prefix})) + # if not specified, "sweep" mode has additional defaults + elif case.get_mode() == 'sweep': + stats = ['maximum', 'minimum', 'percentile', 'samples', 'variance'] + prefixes = ['max', 'min', 'perc', 'samp', 'var'] + for stat, pref in zip(stats, prefixes): + pp_node.append(xmlUtils.newNode(stat, text=case._metric, attrib={'prefix': pref})) + # if not specified, "opt" mode is handled in _modify_inner_optimization_settings def _modify_inner_data_handling(self, template, case): """ @@ -1048,7 +1172,7 @@ def _modify_cash_components(self, template, case, components): tax = subCash._taxable depreciation = subCash._depreciate if subCash._type == 'one-time': - cfNode = xmlUtils.newNode('Capex', text='', attrib={'name':'{name}'.format(name = name), + cfNode = xmlUtils.newNode('Capex', text='', attrib={'name': f'{name}', 'tax':tax, 'inflation': inflation, 'mult_target': mult_target @@ -1061,7 +1185,7 @@ def _modify_cash_components(self, template, case, components): cfNode.append(xmlUtils.newNode('depreciation',attrib={'scheme':'MACRS'}, text = depreciation)) cfs.append(cfNode) else: - cfNode = xmlUtils.newNode('Recurring', text='', attrib={'name':'{name}'.format(name = name), + cfNode = xmlUtils.newNode('Recurring', text='', attrib={'name': f'{name}', 'tax':tax, 'inflation': inflation, 'mult_target': mult_target @@ -1069,7 +1193,7 @@ def _modify_cash_components(self, template, case, components): cfNode.append(xmlUtils.newNode('driver', text = self.namingTemplates['re_cash'].format(period=subCash._period, driverType = driverType, - driverName ='_{comp}_{name}'.format(comp = component.name ,name = name)))) + driverName = f'_{component.name}_{name}'))) cfNode.append(xmlUtils.newNode('alpha',text = '-1.0')) cfs.append(cfNode) @@ -1149,7 +1273,7 @@ def _create_dataobject(self, dataobjects, typ, name, inputs=None, outputs=None, ## if a history set, there better only be one elif typ == 'HistorySet': assert depends is not None - assert len(depends) == 1, 'Depends is: {}'.format(depends) + assert len(depends) == 1, f'Depends is: {depends}' opt = xmlUtils.newNode('options') opt.append(xmlUtils.newNode('pivotParameter', text=list(depends.keys())[0])) new.append(opt) @@ -1217,7 +1341,7 @@ def _iostep_rom_meta(self, template, source): rom_name = source.name # create the output data object objs = template.find('DataObjects') - obj_name = '{}_meta'.format(rom_name) + obj_name = f'{rom_name}_meta' self._create_dataobject(objs, 'DataSet', obj_name) # create the output outstream os_name = obj_name @@ -1261,20 +1385,40 @@ def _build_opt_metric_out_name(self, case): try: # metric name in RAVEN metric_raven_name = case._optimization_settings['metric']['name'] - # potential metric name to add to VariableGroups, DataObjects, Optimizers - opt_out_metric_name = case.optimization_metrics_mapping[metric_raven_name]['prefix'] + # potential metric name to add + opt_out_metric_name = case.metrics_mapping[metric_raven_name]['prefix'] # do I need to add a percent or threshold to this name? if metric_raven_name == 'percentile': opt_out_metric_name += '_' + str(case._optimization_settings['metric']['percent']) - elif metric_raven_name in ['valueAtRisk', 'expectedShortfall']: + elif metric_raven_name in ['valueAtRisk', 'expectedShortfall', 'sortinoRatio', 'gainLossRatio']: opt_out_metric_name += '_' + str(case._optimization_settings['metric']['threshold']) - elif metric_raven_name in ['sortinoRatio', 'gainLossRatio']: - opt_out_metric_name += '_' + case._optimization_settings['metric']['threshold'] - # add target variable to name TODO should this be changeable from NPV? - opt_out_metric_name += '_NPV' + opt_out_metric_name += '_'+case._metric except (TypeError, KeyError): # node not in input file OR # 'metric' is missing from _optimization_settings opt_out_metric_name = 'missing' return opt_out_metric_name + + def _build_result_statistic_names(self, case): + """ + Constructs the names of the statistics requested for output + @ In, case, HERON Case, defining Case instance + @ Out, names, list, list of names of statistics requested for output + """ + names = [] + for name in case._result_statistics: + out_name = case.metrics_mapping[name]['prefix'] + # do I need to add percent or threshold? + if name in ['percentile', 'valueAtRisk', 'expectedShortfall', 'sortinoRatio', 'gainLossRatio']: + # multiple percents or thresholds may be specified + if isinstance(case._result_statistics[name], list): + for attrib in case._result_statistics[name]: + names.append(out_name+'_'+attrib+'_'+case._metric) + else: + names.append(out_name+'_'+case._result_statistics[name]+'_'+case._metric) + else: + out_name += '_'+case._metric + names.append(out_name) + + return names diff --git a/tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv b/tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv index 30313f61..a7c35eca 100644 --- a/tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv +++ b/tests/integration_tests/mechanics/multirun_sweep_opt/gold/Runs_o/opt_soln_0.csv @@ -1,6 +1,6 @@ -iteration,accepted,source_capacity,sink_capacity,mean_NPV -0.0,first,1.05,-2.0,5662.24028348 -1.0,accepted,1.25,-2.0,6741.15327388 -2.0,accepted,1.45,-2.0,7820.12386397 -3.0,accepted,1.85,-2.0,9978.20777236 -4.0,accepted,2.0,-2.0,10787.5312585 +iteration,accepted,source_capacity,sink_capacity,mean_NPV,std_NPV,med_NPV +0.0,first,1.05,-2.0,5662.24028348,0.0,5662.24028348 +1.0,accepted,1.25,-2.0,6741.15327388,1.11389897155e-12,6741.15327388 +2.0,accepted,1.45,-2.0,7820.12386397,1.11389897155e-12,7820.12386397 +3.0,accepted,1.85,-2.0,9978.20777236,0.0,9978.20777236 +4.0,accepted,2.0,-2.0,10787.5312585,0.0,10787.5312585 diff --git a/tests/integration_tests/mechanics/multirun_sweep_opt/heron_input_opt.xml b/tests/integration_tests/mechanics/multirun_sweep_opt/heron_input_opt.xml index 5f73b79b..e5a9b2ed 100644 --- a/tests/integration_tests/mechanics/multirun_sweep_opt/heron_input_opt.xml +++ b/tests/integration_tests/mechanics/multirun_sweep_opt/heron_input_opt.xml @@ -11,7 +11,7 @@ opt - 1 + 3 Time 2 diff --git a/tests/integration_tests/mechanics/optimization_settings/gold/Opt_Runs_o/opt_soln_0.csv b/tests/integration_tests/mechanics/optimization_settings/gold/Opt_Runs_o/opt_soln_0.csv index 876b8687..5b0637ae 100644 --- a/tests/integration_tests/mechanics/optimization_settings/gold/Opt_Runs_o/opt_soln_0.csv +++ b/tests/integration_tests/mechanics/optimization_settings/gold/Opt_Runs_o/opt_soln_0.csv @@ -1,9 +1,9 @@ -iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,VaR_0.05_NPV -0.0,first,1.45,-100.0,-2.0,-2002.5,-22.7414534568 -1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,-62.6657909961 -2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,-69.0824804617 -3.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115 -4.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115 -5.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115 -6.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115 -7.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115 +iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,VaR_0.05_NPV,mean_NPV,std_NPV,med_NPV +0.0,first,1.45,-100.0,-2.0,-2002.5,-22.7414534568,22.7414534595,2.71300300081e-09,22.7414534606 +1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,-62.6657908037,62.665790811,7.47587185876e-09,62.6657908142 +2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,-69.0824804004,69.0824804038,3.94466916627e-09,69.0824804034 +3.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256 +4.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256 +5.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256 +6.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256 +7.0,accepted,10.0,-100.0,-2.0,-2002.5,-77.7220918115,77.7220918232,1.21349097015e-08,77.7220918256 diff --git a/tests/integration_tests/mechanics/result_statistics/gold/Sweep_Runs_o/sweep.csv b/tests/integration_tests/mechanics/result_statistics/gold/Sweep_Runs_o/sweep.csv new file mode 100644 index 00000000..30ec354f --- /dev/null +++ b/tests/integration_tests/mechanics/result_statistics/gold/Sweep_Runs_o/sweep.csv @@ -0,0 +1,3 @@ +BOP_capacity,Electric_Grid_capacity,mean_NPV,std_NPV,med_NPV,min_NPV,max_NPV,var_NPV,perc_5_NPV,perc_95_NPV,varCoeff_NPV,skew_NPV,kurt_NPV,samp_NPV,sharpe_NPV,sortino_median_NPV,glr_median_NPV,es_0.05_NPV,VaR_0.05_NPV,PointProbability,ProbabilityWeight-BOP_capacity,prefix,ProbabilityWeight +1095.0,-1e+200,32365.3088338,3.28481336702e-06,32365.3088333,32365.3088309,32365.3088374,1.07899988562e-11,32365.3088311,32365.308837,1.01491797402e-10,0.714285575694,-1407482650.22,3.0,9853013007.91,22895543491.7,1.6562994969,-32365.3088309,-32365.3088311,0.333333333333,0.5,1,0.5 +1098.0,-1e+200,32453.9809128,3.29381023062e-06,32453.9809123,32453.9809098,32453.9809164,1.08491858354e-11,32453.9809101,32453.980916,1.01491716516e-10,0.714298331064,-2105487593.27,3.0,9853020860.49,22895675664.7,1.65631195711,-32453.9809098,-32453.9809101,0.333333333333,0.5,2,0.5 diff --git a/tests/integration_tests/mechanics/result_statistics/heron_input.xml b/tests/integration_tests/mechanics/result_statistics/heron_input.xml new file mode 100644 index 00000000..badc7bec --- /dev/null +++ b/tests/integration_tests/mechanics/result_statistics/heron_input.xml @@ -0,0 +1,93 @@ + + + result_statistics + dgarrett622 + 2022-08-04 + + Demonstrates returning desired statistics in output. + Based on var_demand_var_price test. + + HERON + + + + sweep + 3 + + Time + 2 + 21 + + + 3 + 0.08 + 0.0 + 0.0 + 50 + + + + + + + + + + + + + + + + + + + + + + + + + + 1095,1098 + + + + 27 + + + + + + + -1e200 + + + + 3 + + + electricity + -1 + + + Speed + + + 1 + + + 1 + + + + + + + + + %HERON%/tests/integration_tests/ARMA/Sine/arma.pk + transfers.py + + + diff --git a/tests/integration_tests/mechanics/result_statistics/tests b/tests/integration_tests/mechanics/result_statistics/tests new file mode 100644 index 00000000..49d88f75 --- /dev/null +++ b/tests/integration_tests/mechanics/result_statistics/tests @@ -0,0 +1,14 @@ +[Tests] + [./ResultStatistics] + type = HeronIntegration + input = heron_input.xml + # prereq = SineArma + [./csv] + type = UnorderedCSV + output = 'Sweep_Runs_o/sweep.csv' + zero_threshold = 1e-6 + rel_err = 1e-5 + [../] + [../] + +[] diff --git a/tests/integration_tests/mechanics/result_statistics/transfers.py b/tests/integration_tests/mechanics/result_statistics/transfers.py new file mode 100644 index 00000000..e665736a --- /dev/null +++ b/tests/integration_tests/mechanics/result_statistics/transfers.py @@ -0,0 +1,23 @@ + +# Copyright 2020, Battelle Energy Alliance, LLC +# ALL RIGHTS RESERVED +""" + Implements transfer functions +""" + +def power_conversion(data, meta): + """ + How to get power from the incoming signal + """ + # get the signal (year, time) from RAVEN ARMA + ## NOTE this behaves completely different if you remove + # the 1.0, and I have no idea why. Leave it there, and + # you get the correct analytic results. + signal = 1.0 * meta['raven_vars']['Signal'][:, :] + # what time step are we currently at? + index = meta['HERON']['time_index'] + # return the entry from the appropriate index + power = signal[index, 0] + 10 + # set the value to return + data['electricity'] = power + return(data, meta) diff --git a/tests/integration_tests/workflows/production_flex_opt/gold/Opt_Runs_o/opt_soln_0.csv b/tests/integration_tests/workflows/production_flex_opt/gold/Opt_Runs_o/opt_soln_0.csv index b1d66ebd..2b9562a0 100644 --- a/tests/integration_tests/workflows/production_flex_opt/gold/Opt_Runs_o/opt_soln_0.csv +++ b/tests/integration_tests/workflows/production_flex_opt/gold/Opt_Runs_o/opt_soln_0.csv @@ -1,7 +1,5 @@ -iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,mean_NPV -0.0,first,1.45,-100.0,-2.0,-2002.5,22.7414534595 -1.0,accepted,2.08639610307,-100.0,-2.0,-2002.5,32.7225393657 -2.0,accepted,2.72279220614,-100.0,-2.0,-2002.5,42.7036221353 -3.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,62.665790811 -4.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,69.0824804038 -5.0,accepted,10.0,-100.0,-2.0,-2002.5,77.7220918232 +iteration,accepted,steamer_capacity,generator_capacity,electr_market_capacity,electr_flex_capacity,mean_NPV,std_NPV,med_NPV +0.0,first,1.45,-100.0,-2.0,-2002.5,22.7414534595,2.71300300081e-09,22.7414534606 +1.0,accepted,3.99558441227,-100.0,-2.0,-2002.5,62.665790811,7.47587185876e-09,62.6657908142 +2.0,accepted,6.54116882454,-100.0,-2.0,-2002.5,69.0824804038,3.94466916627e-09,69.0824804034 +3.0,accepted,10.0,-100.0,-2.0,-2002.5,77.7220918232,1.21349097015e-08,77.7220918256