From 2528989d1ca7f934d648d4819f37de5228b7c8d6 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Wed, 20 Dec 2023 19:35:12 +0100 Subject: [PATCH 01/19] add extended history for AmiciObjective --- pypesto/C.py | 6 ++ pypesto/history/amici.py | 160 ++++++++++++++++++++++++++++++++++ pypesto/history/base.py | 2 +- pypesto/history/generate.py | 9 +- pypesto/optimize/optimizer.py | 3 +- test/base/test_history.py | 32 +++++++ 6 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 pypesto/history/amici.py diff --git a/pypesto/C.py b/pypesto/C.py index 106ca0692..c4b74e34e 100644 --- a/pypesto/C.py +++ b/pypesto/C.py @@ -231,6 +231,12 @@ class InnerParameterType(str, Enum): SUFFIXES_HDF5 = ["hdf5", "h5"] SUFFIXES = SUFFIXES_CSV + SUFFIXES_HDF5 +CPU_TIME_TOTAL = 'cpu_time_total' +PREEQ_CPU_TIME = 'preeq_cpu_time' +PREEQ_CPU_TIME_B = 'preeq_cpu_timeB' +POSTEQ_CPU_TIME = 'posteq_cpu_time' +POSTEQ_CPU_TIME_B = 'posteq_cpu_timeB' + ############################################################################### # PRIOR diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py new file mode 100644 index 000000000..db10efab5 --- /dev/null +++ b/pypesto/history/amici.py @@ -0,0 +1,160 @@ +import time +from pathlib import Path +from typing import Sequence, Union + +import numpy as np + +from ..C import ( + FVAL, + GRAD, + HESS, + N_ITERATIONS, + RES, + SRES, + RDATAS, + TIME, + ModeType, + X, + CPU_TIME_TOTAL, + PREEQ_CPU_TIME, + POSTEQ_CPU_TIME_B, + POSTEQ_CPU_TIME, + PREEQ_CPU_TIME_B +) +from .base import add_fun_from_res, reduce_result_via_options +from .hdf5 import Hdf5History, with_h5_file +from .options import HistoryOptions +from .util import ResultDict, trace_wrap + + +class Hdf5AmiciHistory(Hdf5History): + """ + Stores a representation of the history in an HDF5 file, extended with + AMICI-specific traces of total simulation time, pre-equilibration time and + post-equilibration time + + Parameters + ---------- + id: + Id of the history + file: + HDF5 file name. + options: + History options. Defaults to ``None``. + """ + + def __init__( + self, + id: str, + file: Union[str, Path], + options: Union[HistoryOptions, dict, None] = None, + ): + super().__init__(id, file, options=options) + + @with_h5_file("a") + def _update_trace( + self, + x: np.ndarray, + sensi_orders: tuple[int], + mode: ModeType, + result: ResultDict, + ) -> None: + """Update and possibly store the trace.""" + if not self.options.trace_record: + return + + # calculating function values from residuals + # and reduce via requested history options + result = reduce_result_via_options( + add_fun_from_res(result), self.options + ) + + used_time = time.time() - self.start_time + + values = { + X: x, + FVAL: result[FVAL], + GRAD: result[GRAD], + RES: result[RES], + SRES: result[SRES], + HESS: result[HESS], + TIME: used_time, + CPU_TIME_TOTAL: sum([rdata[CPU_TIME_TOTAL] for rdata in + result[RDATAS]]), + PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in + result[RDATAS]]), + PREEQ_CPU_TIME_B: sum([rdata[PREEQ_CPU_TIME_B] for rdata in + result[RDATAS]]), + POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in + result[RDATAS]]), + POSTEQ_CPU_TIME_B: sum([rdata[POSTEQ_CPU_TIME_B] for rdata in + result[RDATAS]]) + } + + iteration = self._require_group().attrs[N_ITERATIONS] + + for key in values.keys(): + if values[key] is not None: + self._require_group()[f'{iteration}/{key}'] = values[key] + + self._require_group().attrs[N_ITERATIONS] += 1 + + @trace_wrap + def get_cpu_time_total_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative simulation CPU time [ms]. + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return self._get_hdf5_entries(CPU_TIME_TOTAL, ix) + + @trace_wrap + def get_preeq_time_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver [ms]. + (preequilibration) + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return self._get_hdf5_entries(PREEQ_CPU_TIME, ix) + + @trace_wrap + def get_preeq_timeB_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver of the backward + problem [ms] (preequilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return self._get_hdf5_entries(PREEQ_CPU_TIME_B, ix) + + @trace_wrap + def get_posteq_time_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver [ms] + (postequilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return self._get_hdf5_entries(POSTEQ_CPU_TIME, ix) + + @trace_wrap + def get_posteq_timeB_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver of the backward + problem [ms] (postequilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return self._get_hdf5_entries(POSTEQ_CPU_TIME_B, ix) + diff --git a/pypesto/history/base.py b/pypesto/history/base.py index 27f5ef455..8870830ff 100644 --- a/pypesto/history/base.py +++ b/pypesto/history/base.py @@ -249,7 +249,7 @@ def get_time_trace( trim: bool = False, ) -> Union[Sequence[float], float]: """ - Cumulative execution times. + Cumulative execution times [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index 5fdbbfe4c..a9184cc95 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -7,6 +7,7 @@ from .base import CountHistory, HistoryBase from .csv import CsvHistory from .hdf5 import Hdf5History +from .amici import Hdf5AmiciHistory from .memory import MemoryHistory from .options import HistoryOptions from .util import HistoryTypeError @@ -16,6 +17,7 @@ def create_history( id: str, x_names: Sequence[str], options: HistoryOptions, + amici_objective: bool ) -> HistoryBase: """Create a :class:`HistoryBase` object; Factory method. @@ -27,6 +29,8 @@ def create_history( Parameter names. options: History options. + amici_objective: + Indicates if AmiciObjective was used Returns ------- @@ -49,6 +53,9 @@ def create_history( if suffix in SUFFIXES_CSV: return CsvHistory(x_names=x_names, file=storage_file, options=options) elif suffix in SUFFIXES_HDF5: - return Hdf5History(id=id, file=storage_file, options=options) + if amici_objective: + return Hdf5AmiciHistory(id=id, file=storage_file, options=options) + else: + return Hdf5History(id=id, file=storage_file, options=options) else: raise HistoryTypeError(suffix) diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index 52e83a269..ac76d14f8 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -18,7 +18,7 @@ OptimizerHistory, create_history, ) -from ..objective import Objective +from ..objective import AmiciObjective, Objective from ..problem import Problem from ..result import OptimizerResult from .load import fill_result_from_history @@ -106,6 +106,7 @@ def wrapped_minimize( id=id, x_names=[problem.x_names[ix] for ix in problem.x_free_indices], options=history_options, + amici_objective=isinstance(objective, AmiciObjective) ) optimizer_history = OptimizerHistory( history=history, diff --git a/test/base/test_history.py b/test/base/test_history.py index 8338206aa..10b61f473 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -689,6 +689,38 @@ def test_hdf5_history_mp(): ) +def test_hdf5_amici_history(): + objective = load_amici_objective('conversion_reaction')[0] + lb = -2 * np.ones((1, 2)) + ub = 2 * np.ones((1, 2)) + problem = pypesto.Problem( + objective=objective, lb=lb, ub=ub + ) + + optimizer = pypesto.optimize.ScipyOptimizer(options={'maxiter': 10}) + + with tempfile.TemporaryDirectory(dir=".") as tmpdirname: + _, fn = tempfile.mkstemp(".hdf5", dir=f"{tmpdirname}") + + history_options_mp = pypesto.HistoryOptions( + trace_record=True, storage_file=fn + ) + + # optimizing with amici history saved in hdf5 + result = pypesto.optimize.minimize( + problem=problem, + optimizer=optimizer, + n_starts=1, + history_options=history_options_mp, + progress_bar=False, + ) + result.optimize_result.list[0].history.get_cpu_time_total_trace() + result.optimize_result.list[0].history.get_preeq_time_trace() + result.optimize_result.list[0].history.get_preeq_timeB_trace() + result.optimize_result.list[0].history.get_posteq_time_trace() + result.optimize_result.list[0].history.get_posteq_timeB_trace() + + def test_trim_history(): """ Test whether the history gets correctly trimmed to be monotonically From 293f8b762c5bfe080f026634436fc9660ad385c0 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Wed, 20 Dec 2023 19:47:16 +0100 Subject: [PATCH 02/19] test amici history --- pypesto/__init__.py | 1 + pypesto/history/__init__.py | 1 + test/base/test_history.py | 39 +++++++++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pypesto/__init__.py b/pypesto/__init__.py index d1f31749f..25ed3dae4 100644 --- a/pypesto/__init__.py +++ b/pypesto/__init__.py @@ -17,6 +17,7 @@ CountHistoryBase, CsvHistory, Hdf5History, + Hdf5AmiciHistory, NoHistory, HistoryBase, HistoryOptions, diff --git a/pypesto/history/__init__.py b/pypesto/history/__init__.py index 574418a48..ac186b9a0 100644 --- a/pypesto/history/__init__.py +++ b/pypesto/history/__init__.py @@ -8,6 +8,7 @@ and evaluate performance. """ +from .amici import Hdf5AmiciHistory from .base import CountHistory, CountHistoryBase, HistoryBase, NoHistory from .csv import CsvHistory from .generate import create_history diff --git a/test/base/test_history.py b/test/base/test_history.py index 10b61f473..b854faabb 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -16,6 +16,7 @@ from pypesto import ( CsvHistory, Hdf5History, + Hdf5AmiciHistory, HistoryOptions, MemoryHistory, ObjectiveBase, @@ -690,11 +691,17 @@ def test_hdf5_history_mp(): def test_hdf5_amici_history(): - objective = load_amici_objective('conversion_reaction')[0] + objective1 = pypesto.Objective( + fun=so.rosen, grad=so.rosen_der, hess=so.rosen_hess + ) + objective2 = load_amici_objective('conversion_reaction')[0] lb = -2 * np.ones((1, 2)) ub = 2 * np.ones((1, 2)) - problem = pypesto.Problem( - objective=objective, lb=lb, ub=ub + problem1 = pypesto.Problem( + objective=objective1, lb=lb, ub=ub + ) + problem2 = pypesto.Problem( + objective=objective2, lb=lb, ub=ub ) optimizer = pypesto.optimize.ScipyOptimizer(options={'maxiter': 10}) @@ -706,19 +713,31 @@ def test_hdf5_amici_history(): trace_record=True, storage_file=fn ) + result1 = pypesto.optimize.minimize( + problem=problem1, + optimizer=optimizer, + n_starts=1, + history_options=history_options_mp, + progress_bar=False, + ) + assert not isinstance(result1.optimize_result.list[0].history, + Hdf5AmiciHistory) + # optimizing with amici history saved in hdf5 - result = pypesto.optimize.minimize( - problem=problem, + result2 = pypesto.optimize.minimize( + problem=problem2, optimizer=optimizer, n_starts=1, history_options=history_options_mp, progress_bar=False, ) - result.optimize_result.list[0].history.get_cpu_time_total_trace() - result.optimize_result.list[0].history.get_preeq_time_trace() - result.optimize_result.list[0].history.get_preeq_timeB_trace() - result.optimize_result.list[0].history.get_posteq_time_trace() - result.optimize_result.list[0].history.get_posteq_timeB_trace() + assert isinstance(result2.optimize_result.list[0].history, + Hdf5AmiciHistory) + result2.optimize_result.list[0].history.get_cpu_time_total_trace() + result2.optimize_result.list[0].history.get_preeq_time_trace() + result2.optimize_result.list[0].history.get_preeq_timeB_trace() + result2.optimize_result.list[0].history.get_posteq_time_trace() + result2.optimize_result.list[0].history.get_posteq_timeB_trace() def test_trim_history(): From 50f582b35e2f3d62672fbbbbba15fef32155feba Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 13:29:10 +0100 Subject: [PATCH 03/19] add _simulation_to_values method --- pypesto/history/amici.py | 44 ++++++---------------------------------- pypesto/history/hdf5.py | 23 +++++++++++++-------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index db10efab5..0d5d7e987 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -51,34 +51,10 @@ def __init__( ): super().__init__(id, file, options=options) - @with_h5_file("a") - def _update_trace( - self, - x: np.ndarray, - sensi_orders: tuple[int], - mode: ModeType, - result: ResultDict, - ) -> None: - """Update and possibly store the trace.""" - if not self.options.trace_record: - return - - # calculating function values from residuals - # and reduce via requested history options - result = reduce_result_via_options( - add_fun_from_res(result), self.options - ) - - used_time = time.time() - self.start_time - - values = { - X: x, - FVAL: result[FVAL], - GRAD: result[GRAD], - RES: result[RES], - SRES: result[SRES], - HESS: result[HESS], - TIME: used_time, + @staticmethod + def _simulation_to_values(x, result, used_time): + values = Hdf5History._simulation_to_values(x, result, used_time) + values |= { CPU_TIME_TOTAL: sum([rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]]), PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in @@ -88,16 +64,8 @@ def _update_trace( POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]]), POSTEQ_CPU_TIME_B: sum([rdata[POSTEQ_CPU_TIME_B] for rdata in - result[RDATAS]]) - } - - iteration = self._require_group().attrs[N_ITERATIONS] - - for key in values.keys(): - if values[key] is not None: - self._require_group()[f'{iteration}/{key}'] = values[key] - - self._require_group().attrs[N_ITERATIONS] += 1 + result[RDATAS]])} + return values @trace_wrap def get_cpu_time_total_trace( diff --git a/pypesto/history/hdf5.py b/pypesto/history/hdf5.py index a56366ea0..79e3c03e8 100644 --- a/pypesto/history/hdf5.py +++ b/pypesto/history/hdf5.py @@ -302,6 +302,19 @@ def exitflag(self) -> str: except KeyError: return None + @staticmethod + def _simulation_to_values(x, result, used_time): + values = { + X: x, + FVAL: result[FVAL], + GRAD: result[GRAD], + RES: result[RES], + SRES: result[SRES], + HESS: result[HESS], + TIME: used_time + } + return values + @with_h5_file("a") def _update_trace( self, @@ -322,15 +335,7 @@ def _update_trace( used_time = time.time() - self.start_time - values = { - X: x, - FVAL: result[FVAL], - GRAD: result[GRAD], - RES: result[RES], - SRES: result[SRES], - HESS: result[HESS], - TIME: used_time, - } + values = self._simulation_to_values(x, result, used_time) iteration = self._require_group().attrs[N_ITERATIONS] From 29e5f7e5f31747e75236cd5d51849cceeabd10f5 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 16:34:19 +0100 Subject: [PATCH 04/19] add CsvAmiciHistory --- pypesto/__init__.py | 1 + pypesto/history/__init__.py | 2 +- pypesto/history/amici.py | 132 ++++++++++++++++++++++++++++++++---- pypesto/history/csv.py | 43 +++++++----- pypesto/history/generate.py | 9 ++- test/base/test_history.py | 61 +++++++++-------- 6 files changed, 184 insertions(+), 64 deletions(-) diff --git a/pypesto/__init__.py b/pypesto/__init__.py index 25ed3dae4..7bcd97165 100644 --- a/pypesto/__init__.py +++ b/pypesto/__init__.py @@ -16,6 +16,7 @@ CountHistory, CountHistoryBase, CsvHistory, + CsvAmiciHistory, Hdf5History, Hdf5AmiciHistory, NoHistory, diff --git a/pypesto/history/__init__.py b/pypesto/history/__init__.py index ac186b9a0..37fc18377 100644 --- a/pypesto/history/__init__.py +++ b/pypesto/history/__init__.py @@ -8,7 +8,7 @@ and evaluate performance. """ -from .amici import Hdf5AmiciHistory +from .amici import CsvAmiciHistory, Hdf5AmiciHistory from .base import CountHistory, CountHistoryBase, HistoryBase, NoHistory from .csv import CsvHistory from .generate import create_history diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 0d5d7e987..522bdd7db 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -1,37 +1,27 @@ -import time from pathlib import Path from typing import Sequence, Union import numpy as np from ..C import ( - FVAL, - GRAD, - HESS, - N_ITERATIONS, - RES, - SRES, RDATAS, - TIME, - ModeType, - X, CPU_TIME_TOTAL, PREEQ_CPU_TIME, POSTEQ_CPU_TIME_B, POSTEQ_CPU_TIME, PREEQ_CPU_TIME_B ) -from .base import add_fun_from_res, reduce_result_via_options -from .hdf5 import Hdf5History, with_h5_file +from .csv import CsvHistory +from .hdf5 import Hdf5History from .options import HistoryOptions -from .util import ResultDict, trace_wrap +from .util import trace_wrap class Hdf5AmiciHistory(Hdf5History): """ Stores a representation of the history in an HDF5 file, extended with AMICI-specific traces of total simulation time, pre-equilibration time and - post-equilibration time + post-equilibration time. Parameters ---------- @@ -126,3 +116,117 @@ def get_posteq_timeB_trace( """ return self._get_hdf5_entries(POSTEQ_CPU_TIME_B, ix) + +class CsvAmiciHistory(CsvHistory): + """Stores a representation of the history in a CSV file, extended with + AMICI-specific traces of total simulation time, pre-equilibration time and + post-equilibration time. + + Parameters + ---------- + file: + CSV file name. + x_names: + Parameter names. + options: + History options. + load_from_file: + If True, history will be initialized from data in the specified file. + """ + + def __init__( + self, + file: str, + x_names: Sequence[str] = None, + options: Union[HistoryOptions, dict] = None, + load_from_file: bool = False, + ): + super().__init__(file, x_names, options, load_from_file=load_from_file) + + def _trace_columns(self) -> list[tuple]: + columns = super()._trace_columns() + return columns + [ + (c, np.nan) + for c in [ + CPU_TIME_TOTAL, + PREEQ_CPU_TIME, + PREEQ_CPU_TIME_B, + POSTEQ_CPU_TIME, + POSTEQ_CPU_TIME_B + ] + ] + + def _simulation_to_values(self, result, used_time): + values = super()._simulation_to_values(result, used_time) + values |= { + CPU_TIME_TOTAL: sum([rdata[CPU_TIME_TOTAL] for rdata in + result[RDATAS]]), + PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in + result[RDATAS]]), + PREEQ_CPU_TIME_B: sum([rdata[PREEQ_CPU_TIME_B] for rdata in + result[RDATAS]]), + POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in + result[RDATAS]]), + POSTEQ_CPU_TIME_B: sum([rdata[POSTEQ_CPU_TIME_B] for rdata in + result[RDATAS]])} + return values + + @trace_wrap + def get_cpu_time_total_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative simulation CPU time [ms]. + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return list(self._trace[(CPU_TIME_TOTAL, np.nan)].values[ix]) + + @trace_wrap + def get_preeq_time_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver [ms]. + (pre-equilibration) + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return list(self._trace[(PREEQ_CPU_TIME, np.nan)].values[ix]) + + @trace_wrap + def get_preeq_timeB_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver of the backward + problem [ms] (pre-equilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return list(self._trace[(PREEQ_CPU_TIME_B, np.nan)].values[ix]) + + @trace_wrap + def get_posteq_time_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver [ms] + (post-equilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return list(self._trace[(POSTEQ_CPU_TIME, np.nan)].values[ix]) + + @trace_wrap + def get_posteq_timeB_trace( + self, ix: Union[int, Sequence[int], None] = None, trim: bool = False + ) -> Union[Sequence[float], float]: + """ + Cumulative computation time of the steady state solver of the backward + problem [ms] (post-equilibration). + Takes as parameter an index or indices and returns corresponding trace + values. If only a single value is requested, the list is flattened. + """ + return list(self._trace[(POSTEQ_CPU_TIME_B, np.nan)].values[ix]) + diff --git a/pypesto/history/csv.py b/pypesto/history/csv.py index 07df30fbf..9a71c64fe 100644 --- a/pypesto/history/csv.py +++ b/pypesto/history/csv.py @@ -73,7 +73,7 @@ def __init__( # transform strings to np.ndarrays trace[col] = trace[col].apply(string2ndarray) - self._trace = trace + self._trtraceace = trace self.x_names = trace[X].columns self._update_counts_from_trace() @@ -100,6 +100,20 @@ def finalize(self, message: str = None, exitflag: str = None): super().finalize(message=message, exitflag=exitflag) self._save_trace(finalize=True) + def _simulation_to_values(self, result, used_time): + values = { + TIME: used_time, + N_FVAL: self._n_fval, + N_GRAD: self._n_grad, + N_HESS: self._n_hess, + N_RES: self._n_res, + N_SRES: self._n_sres, + FVAL: result[FVAL], + RES: result[RES], + HESS: result[HESS], + } + return values + def _update_trace( self, x: np.ndarray, @@ -127,17 +141,7 @@ def _update_trace( name=len(self._trace), index=self._trace.columns, dtype='object' ) - values = { - TIME: used_time, - N_FVAL: self._n_fval, - N_GRAD: self._n_grad, - N_HESS: self._n_hess, - N_RES: self._n_res, - N_SRES: self._n_sres, - FVAL: result[FVAL], - RES: result[RES], - HESS: result[HESS], - } + values = self._simulation_to_values(result, used_time) for var, val in values.items(): row[(var, np.nan)] = val @@ -162,12 +166,8 @@ def _update_trace( # save trace to file self._save_trace() - def _init_trace(self, x: np.ndarray): - """Initialize the trace.""" - if self.x_names is None: - self.x_names = [f'x{i}' for i, _ in enumerate(x)] - - columns: list[tuple] = [ + def _trace_columns(self) -> list[tuple]: + return [ (c, np.nan) for c in [ TIME, @@ -183,6 +183,13 @@ def _init_trace(self, x: np.ndarray): ] ] + def _init_trace(self, x: np.ndarray): + """Initialize the trace.""" + if self.x_names is None: + self.x_names = [f'x{i}' for i, _ in enumerate(x)] + + columns = self._trace_columns() + for var in [X, GRAD]: if var == X or self.options[f'trace_record_{var}']: columns.extend([(var, x_name) for x_name in self.x_names]) diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index a9184cc95..8f2ce5c7f 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -7,7 +7,7 @@ from .base import CountHistory, HistoryBase from .csv import CsvHistory from .hdf5 import Hdf5History -from .amici import Hdf5AmiciHistory +from .amici import CsvAmiciHistory, Hdf5AmiciHistory from .memory import MemoryHistory from .options import HistoryOptions from .util import HistoryTypeError @@ -51,7 +51,12 @@ def create_history( # create history type based on storage type if suffix in SUFFIXES_CSV: - return CsvHistory(x_names=x_names, file=storage_file, options=options) + if amici_objective: + return CsvAmiciHistory(x_names=x_names, file=storage_file, + options=options) + else: + return CsvHistory(x_names=x_names, file=storage_file, + options=options) elif suffix in SUFFIXES_HDF5: if amici_objective: return Hdf5AmiciHistory(id=id, file=storage_file, options=options) diff --git a/test/base/test_history.py b/test/base/test_history.py index b854faabb..2442c603e 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -15,6 +15,7 @@ import pypesto.optimize as optimize from pypesto import ( CsvHistory, + CsvAmiciHistory, Hdf5History, Hdf5AmiciHistory, HistoryOptions, @@ -707,37 +708,39 @@ def test_hdf5_amici_history(): optimizer = pypesto.optimize.ScipyOptimizer(options={'maxiter': 10}) with tempfile.TemporaryDirectory(dir=".") as tmpdirname: - _, fn = tempfile.mkstemp(".hdf5", dir=f"{tmpdirname}") - - history_options_mp = pypesto.HistoryOptions( - trace_record=True, storage_file=fn - ) + for f_ext, amici_history_class in \ + zip([".csv", ".hdf5"], [CsvAmiciHistory, Hdf5AmiciHistory]): + _, fn = tempfile.mkstemp(f_ext, '{id}', dir=f"{tmpdirname}") - result1 = pypesto.optimize.minimize( - problem=problem1, - optimizer=optimizer, - n_starts=1, - history_options=history_options_mp, - progress_bar=False, - ) - assert not isinstance(result1.optimize_result.list[0].history, - Hdf5AmiciHistory) + history_options_mp = pypesto.HistoryOptions( + trace_record=True, storage_file=fn + ) - # optimizing with amici history saved in hdf5 - result2 = pypesto.optimize.minimize( - problem=problem2, - optimizer=optimizer, - n_starts=1, - history_options=history_options_mp, - progress_bar=False, - ) - assert isinstance(result2.optimize_result.list[0].history, - Hdf5AmiciHistory) - result2.optimize_result.list[0].history.get_cpu_time_total_trace() - result2.optimize_result.list[0].history.get_preeq_time_trace() - result2.optimize_result.list[0].history.get_preeq_timeB_trace() - result2.optimize_result.list[0].history.get_posteq_time_trace() - result2.optimize_result.list[0].history.get_posteq_timeB_trace() + result1 = pypesto.optimize.minimize( + problem=problem1, + optimizer=optimizer, + n_starts=1, + history_options=history_options_mp, + progress_bar=False, + ) + assert not isinstance(result1.optimize_result.list[0].history, + amici_history_class) + + # optimizing with amici history saved in hdf5 + result2 = pypesto.optimize.minimize( + problem=problem2, + optimizer=optimizer, + n_starts=1, + history_options=history_options_mp, + progress_bar=False, + ) + assert isinstance(result2.optimize_result.list[0].history, + amici_history_class) + result2.optimize_result.list[0].history.get_cpu_time_total_trace() + result2.optimize_result.list[0].history.get_preeq_time_trace() + result2.optimize_result.list[0].history.get_preeq_timeB_trace() + result2.optimize_result.list[0].history.get_posteq_time_trace() + result2.optimize_result.list[0].history.get_posteq_timeB_trace() def test_trim_history(): From 4d8d34f8d98a96a4d99a141c7064c73073c4823c Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 17:01:15 +0100 Subject: [PATCH 05/19] docstrings --- pypesto/history/amici.py | 58 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 522bdd7db..d720fac0c 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -4,12 +4,12 @@ import numpy as np from ..C import ( - RDATAS, CPU_TIME_TOTAL, - PREEQ_CPU_TIME, - POSTEQ_CPU_TIME_B, POSTEQ_CPU_TIME, - PREEQ_CPU_TIME_B + POSTEQ_CPU_TIME_B, + PREEQ_CPU_TIME, + PREEQ_CPU_TIME_B, + RDATAS, ) from .csv import CsvHistory from .hdf5 import Hdf5History @@ -19,9 +19,10 @@ class Hdf5AmiciHistory(Hdf5History): """ - Stores a representation of the history in an HDF5 file, extended with - AMICI-specific traces of total simulation time, pre-equilibration time and - post-equilibration time. + Stores history extended by AMICI-specific time traces in an HDF5 file. + + Stores AMICI-specific traces of total simulation time, pre-equilibration + time and post-equilibration time. Parameters ---------- @@ -63,6 +64,7 @@ def get_cpu_time_total_trace( ) -> Union[Sequence[float], float]: """ Cumulative simulation CPU time [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -73,8 +75,8 @@ def get_preeq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver [ms]. - (preequilibration) + Cumulative pre-equilibration time, [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -85,8 +87,8 @@ def get_preeq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver of the backward - problem [ms] (preequilibration). + Cumulative pre-equilibration time of the backward problem, [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -97,8 +99,8 @@ def get_posteq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver [ms] - (postequilibration). + Cumulative post-equilibration time [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -109,8 +111,8 @@ def get_posteq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver of the backward - problem [ms] (postequilibration). + Cumulative post-equilibration time of the backward problem [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -118,9 +120,11 @@ def get_posteq_timeB_trace( class CsvAmiciHistory(CsvHistory): - """Stores a representation of the history in a CSV file, extended with - AMICI-specific traces of total simulation time, pre-equilibration time and - post-equilibration time. + """ + Stores history extended by AMICI-specific time traces in a CSV file. + + Stores AMICI-specific traces of total simulation time, pre-equilibration + time and post-equilibration time. Parameters ---------- @@ -177,6 +181,7 @@ def get_cpu_time_total_trace( ) -> Union[Sequence[float], float]: """ Cumulative simulation CPU time [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -187,8 +192,8 @@ def get_preeq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver [ms]. - (pre-equilibration) + Cumulative pre-equilibration time [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -199,8 +204,8 @@ def get_preeq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver of the backward - problem [ms] (pre-equilibration). + Cumulative pre-equilibration time of the backward problem [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -211,8 +216,8 @@ def get_posteq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver [ms] - (post-equilibration). + Cumulative post-equilibration time [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ @@ -223,10 +228,9 @@ def get_posteq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative computation time of the steady state solver of the backward - problem [ms] (post-equilibration). + Cumulative post-equilibration time of the backward problem [ms]. + Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ return list(self._trace[(POSTEQ_CPU_TIME_B, np.nan)].values[ix]) - From 079dbd6f2e091478eb04e843335b508d235d20e1 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 17:05:32 +0100 Subject: [PATCH 06/19] fix typo --- pypesto/history/csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypesto/history/csv.py b/pypesto/history/csv.py index 9a71c64fe..c672fe8fe 100644 --- a/pypesto/history/csv.py +++ b/pypesto/history/csv.py @@ -73,7 +73,7 @@ def __init__( # transform strings to np.ndarrays trace[col] = trace[col].apply(string2ndarray) - self._trtraceace = trace + self._trace = trace self.x_names = trace[X].columns self._update_counts_from_trace() From 660655898f9dd3e179757fe2f403c1863a4f069c Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 17:13:02 +0100 Subject: [PATCH 07/19] spell out Backward --- pypesto/C.py | 4 ++-- pypesto/history/amici.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pypesto/C.py b/pypesto/C.py index c4b74e34e..136f3d0f9 100644 --- a/pypesto/C.py +++ b/pypesto/C.py @@ -233,9 +233,9 @@ class InnerParameterType(str, Enum): CPU_TIME_TOTAL = 'cpu_time_total' PREEQ_CPU_TIME = 'preeq_cpu_time' -PREEQ_CPU_TIME_B = 'preeq_cpu_timeB' +PREEQ_CPU_TIME_BACKWARD = 'preeq_cpu_timeB' POSTEQ_CPU_TIME = 'posteq_cpu_time' -POSTEQ_CPU_TIME_B = 'posteq_cpu_timeB' +POSTEQ_CPU_TIME_BACKWARD = 'posteq_cpu_timeB' ############################################################################### diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index d720fac0c..129005f37 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -6,9 +6,9 @@ from ..C import ( CPU_TIME_TOTAL, POSTEQ_CPU_TIME, - POSTEQ_CPU_TIME_B, + POSTEQ_CPU_TIME_BACKWARD, PREEQ_CPU_TIME, - PREEQ_CPU_TIME_B, + PREEQ_CPU_TIME_BACKWARD, RDATAS, ) from .csv import CsvHistory @@ -50,12 +50,12 @@ def _simulation_to_values(x, result, used_time): result[RDATAS]]), PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]]), - PREEQ_CPU_TIME_B: sum([rdata[PREEQ_CPU_TIME_B] for rdata in - result[RDATAS]]), + PREEQ_CPU_TIME_BACKWARD: sum([rdata[PREEQ_CPU_TIME_BACKWARD] for + rdata in result[RDATAS]]), POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]]), - POSTEQ_CPU_TIME_B: sum([rdata[POSTEQ_CPU_TIME_B] for rdata in - result[RDATAS]])} + POSTEQ_CPU_TIME_BACKWARD: sum([rdata[POSTEQ_CPU_TIME_BACKWARD] for + rdata in result[RDATAS]])} return values @trace_wrap @@ -92,7 +92,7 @@ def get_preeq_timeB_trace( Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ - return self._get_hdf5_entries(PREEQ_CPU_TIME_B, ix) + return self._get_hdf5_entries(PREEQ_CPU_TIME_BACKWARD, ix) @trace_wrap def get_posteq_time_trace( @@ -116,7 +116,7 @@ def get_posteq_timeB_trace( Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ - return self._get_hdf5_entries(POSTEQ_CPU_TIME_B, ix) + return self._get_hdf5_entries(POSTEQ_CPU_TIME_BACKWARD, ix) class CsvAmiciHistory(CsvHistory): @@ -154,9 +154,9 @@ def _trace_columns(self) -> list[tuple]: for c in [ CPU_TIME_TOTAL, PREEQ_CPU_TIME, - PREEQ_CPU_TIME_B, + PREEQ_CPU_TIME_BACKWARD, POSTEQ_CPU_TIME, - POSTEQ_CPU_TIME_B + POSTEQ_CPU_TIME_BACKWARD ] ] @@ -167,12 +167,12 @@ def _simulation_to_values(self, result, used_time): result[RDATAS]]), PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]]), - PREEQ_CPU_TIME_B: sum([rdata[PREEQ_CPU_TIME_B] for rdata in - result[RDATAS]]), + PREEQ_CPU_TIME_BACKWARD: sum([rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in + result[RDATAS]]), POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]]), - POSTEQ_CPU_TIME_B: sum([rdata[POSTEQ_CPU_TIME_B] for rdata in - result[RDATAS]])} + POSTEQ_CPU_TIME_BACKWARD: sum([rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in + result[RDATAS]])} return values @trace_wrap @@ -209,7 +209,7 @@ def get_preeq_timeB_trace( Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ - return list(self._trace[(PREEQ_CPU_TIME_B, np.nan)].values[ix]) + return list(self._trace[(PREEQ_CPU_TIME_BACKWARD, np.nan)].values[ix]) @trace_wrap def get_posteq_time_trace( @@ -233,4 +233,4 @@ def get_posteq_timeB_trace( Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. """ - return list(self._trace[(POSTEQ_CPU_TIME_B, np.nan)].values[ix]) + return list(self._trace[(POSTEQ_CPU_TIME_BACKWARD, np.nan)].values[ix]) From ec778619f798eb2cfd3a48f05890734ba34b65e7 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 22 Dec 2023 17:49:32 +0100 Subject: [PATCH 08/19] import order --- pypesto/history/generate.py | 2 +- test/base/test_history.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index 8f2ce5c7f..29ca024cb 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -4,10 +4,10 @@ from typing import Sequence from ..C import SUFFIXES_CSV, SUFFIXES_HDF5 +from .amici import CsvAmiciHistory, Hdf5AmiciHistory from .base import CountHistory, HistoryBase from .csv import CsvHistory from .hdf5 import Hdf5History -from .amici import CsvAmiciHistory, Hdf5AmiciHistory from .memory import MemoryHistory from .options import HistoryOptions from .util import HistoryTypeError diff --git a/test/base/test_history.py b/test/base/test_history.py index 2442c603e..061ce3bdc 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -14,10 +14,10 @@ import pypesto import pypesto.optimize as optimize from pypesto import ( - CsvHistory, CsvAmiciHistory, - Hdf5History, + CsvHistory, Hdf5AmiciHistory, + Hdf5History, HistoryOptions, MemoryHistory, ObjectiveBase, From 1bd4fc791a2f4a36c15cdc6a7184d4d4aaa6f67b Mon Sep 17 00:00:00 2001 From: Maren Philipps Date: Fri, 5 Jan 2024 10:06:21 +0100 Subject: [PATCH 09/19] black --- pypesto/history/amici.py | 54 +++++++++++++++++++++-------------- pypesto/history/generate.py | 12 ++++---- pypesto/history/hdf5.py | 2 +- pypesto/optimize/optimizer.py | 2 +- test/base/test_history.py | 23 +++++++-------- 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 129005f37..539028954 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -46,16 +46,22 @@ def __init__( def _simulation_to_values(x, result, used_time): values = Hdf5History._simulation_to_values(x, result, used_time) values |= { - CPU_TIME_TOTAL: sum([rdata[CPU_TIME_TOTAL] for rdata in - result[RDATAS]]), - PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in - result[RDATAS]]), - PREEQ_CPU_TIME_BACKWARD: sum([rdata[PREEQ_CPU_TIME_BACKWARD] for - rdata in result[RDATAS]]), - POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in - result[RDATAS]]), - POSTEQ_CPU_TIME_BACKWARD: sum([rdata[POSTEQ_CPU_TIME_BACKWARD] for - rdata in result[RDATAS]])} + CPU_TIME_TOTAL: sum( + [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] + ), + PREEQ_CPU_TIME: sum( + [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] + ), + PREEQ_CPU_TIME_BACKWARD: sum( + [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + ), + POSTEQ_CPU_TIME: sum( + [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] + ), + POSTEQ_CPU_TIME_BACKWARD: sum( + [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + ), + } return values @trace_wrap @@ -156,23 +162,29 @@ def _trace_columns(self) -> list[tuple]: PREEQ_CPU_TIME, PREEQ_CPU_TIME_BACKWARD, POSTEQ_CPU_TIME, - POSTEQ_CPU_TIME_BACKWARD + POSTEQ_CPU_TIME_BACKWARD, ] ] def _simulation_to_values(self, result, used_time): values = super()._simulation_to_values(result, used_time) values |= { - CPU_TIME_TOTAL: sum([rdata[CPU_TIME_TOTAL] for rdata in - result[RDATAS]]), - PREEQ_CPU_TIME: sum([rdata[PREEQ_CPU_TIME] for rdata in - result[RDATAS]]), - PREEQ_CPU_TIME_BACKWARD: sum([rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in - result[RDATAS]]), - POSTEQ_CPU_TIME: sum([rdata[POSTEQ_CPU_TIME] for rdata in - result[RDATAS]]), - POSTEQ_CPU_TIME_BACKWARD: sum([rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in - result[RDATAS]])} + CPU_TIME_TOTAL: sum( + [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] + ), + PREEQ_CPU_TIME: sum( + [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] + ), + PREEQ_CPU_TIME_BACKWARD: sum( + [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + ), + POSTEQ_CPU_TIME: sum( + [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] + ), + POSTEQ_CPU_TIME_BACKWARD: sum( + [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + ), + } return values @trace_wrap diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index 29ca024cb..acde09838 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -17,7 +17,7 @@ def create_history( id: str, x_names: Sequence[str], options: HistoryOptions, - amici_objective: bool + amici_objective: bool, ) -> HistoryBase: """Create a :class:`HistoryBase` object; Factory method. @@ -52,11 +52,13 @@ def create_history( # create history type based on storage type if suffix in SUFFIXES_CSV: if amici_objective: - return CsvAmiciHistory(x_names=x_names, file=storage_file, - options=options) + return CsvAmiciHistory( + x_names=x_names, file=storage_file, options=options + ) else: - return CsvHistory(x_names=x_names, file=storage_file, - options=options) + return CsvHistory( + x_names=x_names, file=storage_file, options=options + ) elif suffix in SUFFIXES_HDF5: if amici_objective: return Hdf5AmiciHistory(id=id, file=storage_file, options=options) diff --git a/pypesto/history/hdf5.py b/pypesto/history/hdf5.py index 79e3c03e8..4d05cbd97 100644 --- a/pypesto/history/hdf5.py +++ b/pypesto/history/hdf5.py @@ -311,7 +311,7 @@ def _simulation_to_values(x, result, used_time): RES: result[RES], SRES: result[SRES], HESS: result[HESS], - TIME: used_time + TIME: used_time, } return values diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index ac76d14f8..bf6817e04 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -106,7 +106,7 @@ def wrapped_minimize( id=id, x_names=[problem.x_names[ix] for ix in problem.x_free_indices], options=history_options, - amici_objective=isinstance(objective, AmiciObjective) + amici_objective=isinstance(objective, AmiciObjective), ) optimizer_history = OptimizerHistory( history=history, diff --git a/test/base/test_history.py b/test/base/test_history.py index 061ce3bdc..bab88d94f 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -698,18 +698,15 @@ def test_hdf5_amici_history(): objective2 = load_amici_objective('conversion_reaction')[0] lb = -2 * np.ones((1, 2)) ub = 2 * np.ones((1, 2)) - problem1 = pypesto.Problem( - objective=objective1, lb=lb, ub=ub - ) - problem2 = pypesto.Problem( - objective=objective2, lb=lb, ub=ub - ) + problem1 = pypesto.Problem(objective=objective1, lb=lb, ub=ub) + problem2 = pypesto.Problem(objective=objective2, lb=lb, ub=ub) optimizer = pypesto.optimize.ScipyOptimizer(options={'maxiter': 10}) with tempfile.TemporaryDirectory(dir=".") as tmpdirname: - for f_ext, amici_history_class in \ - zip([".csv", ".hdf5"], [CsvAmiciHistory, Hdf5AmiciHistory]): + for f_ext, amici_history_class in zip( + [".csv", ".hdf5"], [CsvAmiciHistory, Hdf5AmiciHistory] + ): _, fn = tempfile.mkstemp(f_ext, '{id}', dir=f"{tmpdirname}") history_options_mp = pypesto.HistoryOptions( @@ -723,8 +720,9 @@ def test_hdf5_amici_history(): history_options=history_options_mp, progress_bar=False, ) - assert not isinstance(result1.optimize_result.list[0].history, - amici_history_class) + assert not isinstance( + result1.optimize_result.list[0].history, amici_history_class + ) # optimizing with amici history saved in hdf5 result2 = pypesto.optimize.minimize( @@ -734,8 +732,9 @@ def test_hdf5_amici_history(): history_options=history_options_mp, progress_bar=False, ) - assert isinstance(result2.optimize_result.list[0].history, - amici_history_class) + assert isinstance( + result2.optimize_result.list[0].history, amici_history_class + ) result2.optimize_result.list[0].history.get_cpu_time_total_trace() result2.optimize_result.list[0].history.get_preeq_time_trace() result2.optimize_result.list[0].history.get_preeq_timeB_trace() From 92b8c5e536eb0c3b715f4361bb7736890e15e0b6 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Fri, 5 Jan 2024 11:38:35 +0100 Subject: [PATCH 10/19] convert ms to s --- pypesto/history/amici.py | 42 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 539028954..41a57a7be 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -45,22 +45,23 @@ def __init__( @staticmethod def _simulation_to_values(x, result, used_time): values = Hdf5History._simulation_to_values(x, result, used_time) + # default unit for time in amici is [ms], converted to [s] values |= { CPU_TIME_TOTAL: sum( [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - ), + )*0.001, PREEQ_CPU_TIME: sum( [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - ), + )*0.001, PREEQ_CPU_TIME_BACKWARD: sum( [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ), + )*0.001, POSTEQ_CPU_TIME: sum( [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - ), + )*0.001, POSTEQ_CPU_TIME_BACKWARD: sum( [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ), + )*0.001, } return values @@ -69,7 +70,7 @@ def get_cpu_time_total_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative simulation CPU time [ms]. + Cumulative simulation CPU time [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -81,7 +82,7 @@ def get_preeq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative pre-equilibration time, [ms]. + Cumulative pre-equilibration time, [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -93,7 +94,7 @@ def get_preeq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative pre-equilibration time of the backward problem, [ms]. + Cumulative pre-equilibration time of the backward problem, [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -105,7 +106,7 @@ def get_posteq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative post-equilibration time [ms]. + Cumulative post-equilibration time [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -117,7 +118,7 @@ def get_posteq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative post-equilibration time of the backward problem [ms]. + Cumulative post-equilibration time of the backward problem [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -168,22 +169,23 @@ def _trace_columns(self) -> list[tuple]: def _simulation_to_values(self, result, used_time): values = super()._simulation_to_values(result, used_time) + # default unit for time in amici is [ms], converted to [s] values |= { CPU_TIME_TOTAL: sum( [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - ), + )*0.001, PREEQ_CPU_TIME: sum( [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - ), + )*0.001, PREEQ_CPU_TIME_BACKWARD: sum( [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ), + )*0.001, POSTEQ_CPU_TIME: sum( [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - ), + )*0.001, POSTEQ_CPU_TIME_BACKWARD: sum( [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ), + )*0.001, } return values @@ -192,7 +194,7 @@ def get_cpu_time_total_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative simulation CPU time [ms]. + Cumulative simulation CPU time [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -204,7 +206,7 @@ def get_preeq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative pre-equilibration time [ms]. + Cumulative pre-equilibration time [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -216,7 +218,7 @@ def get_preeq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative pre-equilibration time of the backward problem [ms]. + Cumulative pre-equilibration time of the backward problem [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -228,7 +230,7 @@ def get_posteq_time_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative post-equilibration time [ms]. + Cumulative post-equilibration time [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. @@ -240,7 +242,7 @@ def get_posteq_timeB_trace( self, ix: Union[int, Sequence[int], None] = None, trim: bool = False ) -> Union[Sequence[float], float]: """ - Cumulative post-equilibration time of the backward problem [ms]. + Cumulative post-equilibration time of the backward problem [s]. Takes as parameter an index or indices and returns corresponding trace values. If only a single value is requested, the list is flattened. From ed63dfe98c3f54de6a3aac29d7c4142acd6abf7e Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 11:38:38 +0100 Subject: [PATCH 11/19] no amici in pypesto.optimize --- pypesto/history/generate.py | 22 +++++---------------- pypesto/objective/amici/amici.py | 34 +++++++++++++++++++++++++++++++- pypesto/objective/base.py | 5 ++++- pypesto/optimize/optimizer.py | 5 ++--- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index acde09838..b6dd0e636 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -4,7 +4,6 @@ from typing import Sequence from ..C import SUFFIXES_CSV, SUFFIXES_HDF5 -from .amici import CsvAmiciHistory, Hdf5AmiciHistory from .base import CountHistory, HistoryBase from .csv import CsvHistory from .hdf5 import Hdf5History @@ -16,8 +15,7 @@ def create_history( id: str, x_names: Sequence[str], - options: HistoryOptions, - amici_objective: bool, + options: HistoryOptions ) -> HistoryBase: """Create a :class:`HistoryBase` object; Factory method. @@ -29,8 +27,6 @@ def create_history( Parameter names. options: History options. - amici_objective: - Indicates if AmiciObjective was used Returns ------- @@ -51,18 +47,10 @@ def create_history( # create history type based on storage type if suffix in SUFFIXES_CSV: - if amici_objective: - return CsvAmiciHistory( - x_names=x_names, file=storage_file, options=options - ) - else: - return CsvHistory( - x_names=x_names, file=storage_file, options=options - ) + return CsvHistory( + x_names=x_names, file=storage_file, options=options + ) elif suffix in SUFFIXES_HDF5: - if amici_objective: - return Hdf5AmiciHistory(id=id, file=storage_file, options=options) - else: - return Hdf5History(id=id, file=storage_file, options=options) + return Hdf5History(id=id, file=storage_file, options=options) else: raise HistoryTypeError(suffix) diff --git a/pypesto/objective/amici/amici.py b/pypesto/objective/amici/amici.py index 521dba3b8..685a60539 100644 --- a/pypesto/objective/amici/amici.py +++ b/pypesto/objective/amici/amici.py @@ -3,11 +3,15 @@ import os import tempfile from collections import OrderedDict +from pathlib import Path from typing import TYPE_CHECKING, Dict, Optional, Sequence, Tuple, Union import numpy as np -from ...C import FVAL, INNER_PARAMETERS, MODE_FUN, MODE_RES, RDATAS, ModeType +from ...C import (FVAL, INNER_PARAMETERS, MODE_FUN, MODE_RES, RDATAS, + ModeType, SUFFIXES_CSV, SUFFIXES_HDF5) +from ...history import (CountHistory, CsvAmiciHistory, Hdf5AmiciHistory, + HistoryOptions, MemoryHistory, HistoryTypeError) from ..base import ObjectiveBase, ResultDict from .amici_calculator import AmiciCalculator from .amici_util import ( @@ -227,6 +231,34 @@ def get_config(self) -> dict: return info + def create_history(self, + id: str, + x_names: Sequence[str], + options: HistoryOptions): + """See `history.generate.create_history` documentation.""" + # create different history types based on the inputs + if options.storage_file is None: + if options.trace_record: + return MemoryHistory(options=options) + else: + return CountHistory(options=options) + + # replace id template in storage file + storage_file = options.storage_file.replace("{id}", id) + + # evaluate type + suffix = Path(storage_file).suffix[1:] + + # create history type based on storage type + if suffix in SUFFIXES_CSV: + return CsvAmiciHistory( + x_names=x_names, file=storage_file, options=options + ) + elif suffix in SUFFIXES_HDF5: + return Hdf5AmiciHistory(id=id, file=storage_file, options=options) + else: + raise HistoryTypeError(suffix) + def initialize(self): """See `ObjectiveBase` documentation.""" super().initialize() diff --git a/pypesto/objective/base.py b/pypesto/objective/base.py index 3e47f0ef7..5956371e6 100644 --- a/pypesto/objective/base.py +++ b/pypesto/objective/base.py @@ -7,7 +7,7 @@ import pandas as pd from ..C import FVAL, GRAD, HESS, MODE_FUN, MODE_RES, RES, SRES, ModeType -from ..history import NoHistory +from ..history import NoHistory, create_history from .pre_post_process import FixedParametersProcessor, PrePostProcessor ResultDict = Dict[str, Union[float, np.ndarray, Dict]] @@ -119,6 +119,9 @@ def initialize(self): By default does nothing. """ + def create_history(self, id, x_names, options): + return create_history(id, x_names, options) + def __call__( self, x: np.ndarray, diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index bf6817e04..9d102bc23 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -102,11 +102,10 @@ def wrapped_minimize( objective.initialize() # initialize the history - history = create_history( + history = objective.create_history( id=id, x_names=[problem.x_names[ix] for ix in problem.x_free_indices], - options=history_options, - amici_objective=isinstance(objective, AmiciObjective), + options=history_options ) optimizer_history = OptimizerHistory( history=history, From 8c0fdfb07da7666e861434cc83ad9bc4d9d31bbc Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 11:51:13 +0100 Subject: [PATCH 12/19] Update test/base/test_history.py Co-authored-by: Daniel Weindl --- test/base/test_history.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/test/base/test_history.py b/test/base/test_history.py index bab88d94f..5430624f3 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -732,14 +732,25 @@ def test_hdf5_amici_history(): history_options=history_options_mp, progress_bar=False, ) - assert isinstance( - result2.optimize_result.list[0].history, amici_history_class + history = result2.optimize_result.list[0].history + assert isinstance(history, amici_history_class) + + assert np.all( + history.get_cpu_time_total_trace() + >= history.get_preeq_time_trace() + ) + assert np.all( + history.get_cpu_time_total_trace() + >= history.get_preeq_timeB_trace() + ) + assert np.all( + history.get_cpu_time_total_trace() + >= history.get_posteq_time_trace() + ) + assert np.all( + history.get_cpu_time_total_trace() + >= history.get_posteq_timeB_trace() ) - result2.optimize_result.list[0].history.get_cpu_time_total_trace() - result2.optimize_result.list[0].history.get_preeq_time_trace() - result2.optimize_result.list[0].history.get_preeq_timeB_trace() - result2.optimize_result.list[0].history.get_posteq_time_trace() - result2.optimize_result.list[0].history.get_posteq_timeB_trace() def test_trim_history(): From 26059db703a06154e62751cad8efa07ed511f84e Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 12:00:38 +0100 Subject: [PATCH 13/19] Update pypesto/optimize/optimizer.py Co-authored-by: Daniel Weindl --- pypesto/optimize/optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index 9d102bc23..8d2ee0213 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -18,7 +18,7 @@ OptimizerHistory, create_history, ) -from ..objective import AmiciObjective, Objective +from ..objective import Objective from ..problem import Problem from ..result import OptimizerResult from .load import fill_result_from_history From 926620b2d1abfae5d2e88992c90eebb2410bf124 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 12:13:48 +0100 Subject: [PATCH 14/19] delete files in the test --- pypesto/history/generate.py | 4 +--- test/base/test_history.py | 8 +++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index b6dd0e636..1772f7af1 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -47,9 +47,7 @@ def create_history( # create history type based on storage type if suffix in SUFFIXES_CSV: - return CsvHistory( - x_names=x_names, file=storage_file, options=options - ) + return CsvHistory(x_names=x_names, file=storage_file, options=options) elif suffix in SUFFIXES_HDF5: return Hdf5History(id=id, file=storage_file, options=options) else: diff --git a/test/base/test_history.py b/test/base/test_history.py index 5430624f3..070231526 100644 --- a/test/base/test_history.py +++ b/test/base/test_history.py @@ -709,7 +709,7 @@ def test_hdf5_amici_history(): ): _, fn = tempfile.mkstemp(f_ext, '{id}', dir=f"{tmpdirname}") - history_options_mp = pypesto.HistoryOptions( + history_options = pypesto.HistoryOptions( trace_record=True, storage_file=fn ) @@ -717,19 +717,21 @@ def test_hdf5_amici_history(): problem=problem1, optimizer=optimizer, n_starts=1, - history_options=history_options_mp, + history_options=history_options, progress_bar=False, ) assert not isinstance( result1.optimize_result.list[0].history, amici_history_class ) + os.remove(fn) + os.remove(fn.replace('{id}', '0')) # optimizing with amici history saved in hdf5 result2 = pypesto.optimize.minimize( problem=problem2, optimizer=optimizer, n_starts=1, - history_options=history_options_mp, + history_options=history_options, progress_bar=False, ) history = result2.optimize_result.list[0].history From 13886934827f494098e6ce1a69c1b5b20fabb724 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 12:24:10 +0100 Subject: [PATCH 15/19] style --- pypesto/objective/base.py | 1 + pypesto/optimize/optimizer.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pypesto/objective/base.py b/pypesto/objective/base.py index 5956371e6..a38114bd4 100644 --- a/pypesto/objective/base.py +++ b/pypesto/objective/base.py @@ -120,6 +120,7 @@ def initialize(self): """ def create_history(self, id, x_names, options): + """See `history.generate.create_history` documentation.""" return create_history(id, x_names, options) def __call__( diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index 8d2ee0213..1a582c242 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -15,8 +15,7 @@ from ..history import ( HistoryOptions, NoHistory, - OptimizerHistory, - create_history, + OptimizerHistory ) from ..objective import Objective from ..problem import Problem From f5430412ec34b5218142f1d77c8dfe2bf92d7357 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 12:37:39 +0100 Subject: [PATCH 16/19] isort --- pypesto/objective/amici/amici.py | 22 ++++++++++++++++++---- pypesto/optimize/optimizer.py | 6 +----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pypesto/objective/amici/amici.py b/pypesto/objective/amici/amici.py index 685a60539..f1d359645 100644 --- a/pypesto/objective/amici/amici.py +++ b/pypesto/objective/amici/amici.py @@ -8,10 +8,24 @@ import numpy as np -from ...C import (FVAL, INNER_PARAMETERS, MODE_FUN, MODE_RES, RDATAS, - ModeType, SUFFIXES_CSV, SUFFIXES_HDF5) -from ...history import (CountHistory, CsvAmiciHistory, Hdf5AmiciHistory, - HistoryOptions, MemoryHistory, HistoryTypeError) +from ...C import ( + FVAL, + INNER_PARAMETERS, + MODE_FUN, + MODE_RES, + RDATAS, + SUFFIXES_CSV, + SUFFIXES_HDF5, + ModeType, +) +from ...history import ( + CountHistory, + CsvAmiciHistory, + Hdf5AmiciHistory, + HistoryOptions, + HistoryTypeError, + MemoryHistory, +) from ..base import ObjectiveBase, ResultDict from .amici_calculator import AmiciCalculator from .amici_util import ( diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index 1a582c242..f54f66570 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -12,11 +12,7 @@ import scipy.optimize from ..C import FVAL, GRAD, INNER_PARAMETERS, MODE_FUN, MODE_RES -from ..history import ( - HistoryOptions, - NoHistory, - OptimizerHistory -) +from ..history import HistoryOptions, NoHistory, OptimizerHistory from ..objective import Objective from ..problem import Problem from ..result import OptimizerResult From ddeee94ec4ec38c45f61f92d640f371478741b31 Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Mon, 8 Jan 2024 14:20:34 +0100 Subject: [PATCH 17/19] black --- pypesto/history/amici.py | 30 ++++++++++++++++++++---------- pypesto/history/generate.py | 4 +--- pypesto/objective/amici/amici.py | 7 +++---- pypesto/optimize/optimizer.py | 2 +- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 41a57a7be..6119348c8 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -49,19 +49,24 @@ def _simulation_to_values(x, result, used_time): values |= { CPU_TIME_TOTAL: sum( [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, PREEQ_CPU_TIME: sum( [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, PREEQ_CPU_TIME_BACKWARD: sum( [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, POSTEQ_CPU_TIME: sum( [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, POSTEQ_CPU_TIME_BACKWARD: sum( [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, } return values @@ -173,19 +178,24 @@ def _simulation_to_values(self, result, used_time): values |= { CPU_TIME_TOTAL: sum( [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, PREEQ_CPU_TIME: sum( [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, PREEQ_CPU_TIME_BACKWARD: sum( [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, POSTEQ_CPU_TIME: sum( [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, POSTEQ_CPU_TIME_BACKWARD: sum( [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - )*0.001, + ) + * 0.001, } return values diff --git a/pypesto/history/generate.py b/pypesto/history/generate.py index 1772f7af1..3e291c7c7 100644 --- a/pypesto/history/generate.py +++ b/pypesto/history/generate.py @@ -13,9 +13,7 @@ def create_history( - id: str, - x_names: Sequence[str], - options: HistoryOptions + id: str, x_names: Sequence[str], options: HistoryOptions ) -> HistoryBase: """Create a :class:`HistoryBase` object; Factory method. diff --git a/pypesto/objective/amici/amici.py b/pypesto/objective/amici/amici.py index f1d359645..13e2d87e8 100644 --- a/pypesto/objective/amici/amici.py +++ b/pypesto/objective/amici/amici.py @@ -245,10 +245,9 @@ def get_config(self) -> dict: return info - def create_history(self, - id: str, - x_names: Sequence[str], - options: HistoryOptions): + def create_history( + self, id: str, x_names: Sequence[str], options: HistoryOptions + ): """See `history.generate.create_history` documentation.""" # create different history types based on the inputs if options.storage_file is None: diff --git a/pypesto/optimize/optimizer.py b/pypesto/optimize/optimizer.py index f54f66570..166020958 100644 --- a/pypesto/optimize/optimizer.py +++ b/pypesto/optimize/optimizer.py @@ -100,7 +100,7 @@ def wrapped_minimize( history = objective.create_history( id=id, x_names=[problem.x_names[ix] for ix in problem.x_free_indices], - options=history_options + options=history_options, ) optimizer_history = OptimizerHistory( history=history, From 994900cebd4316fcc232c49e6a216dc0730db17c Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Tue, 9 Jan 2024 12:20:03 +0100 Subject: [PATCH 18/19] Update pypesto/history/amici.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fabian Fröhlich --- pypesto/history/amici.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 6119348c8..6051ebbfd 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -47,6 +47,11 @@ def _simulation_to_values(x, result, used_time): values = Hdf5History._simulation_to_values(x, result, used_time) # default unit for time in amici is [ms], converted to [s] values |= { + key: sum( + [rdata[key] for rdata in result[RDATAS]] + ) * 0.001 + for key in (CPU_TIME_TOTAL, PREEQ_CPU_TIME, PREEQ_CPU_TIME_BACKWARD, POSTEQ_CPU_TIME, POSTEQ_CPU_TIME_BACKWARD) + } CPU_TIME_TOTAL: sum( [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] ) From fce00372be5cf07fd781afe904549029dbeabaec Mon Sep 17 00:00:00 2001 From: Polina Lakrisenko Date: Tue, 9 Jan 2024 12:27:48 +0100 Subject: [PATCH 19/19] more compact code --- pypesto/history/amici.py | 57 ++++++++++------------------------------ 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/pypesto/history/amici.py b/pypesto/history/amici.py index 6051ebbfd..cfd330cd5 100644 --- a/pypesto/history/amici.py +++ b/pypesto/history/amici.py @@ -47,31 +47,14 @@ def _simulation_to_values(x, result, used_time): values = Hdf5History._simulation_to_values(x, result, used_time) # default unit for time in amici is [ms], converted to [s] values |= { - key: sum( - [rdata[key] for rdata in result[RDATAS]] - ) * 0.001 - for key in (CPU_TIME_TOTAL, PREEQ_CPU_TIME, PREEQ_CPU_TIME_BACKWARD, POSTEQ_CPU_TIME, POSTEQ_CPU_TIME_BACKWARD) - } - CPU_TIME_TOTAL: sum( - [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - ) - * 0.001, - PREEQ_CPU_TIME: sum( - [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - ) - * 0.001, - PREEQ_CPU_TIME_BACKWARD: sum( - [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ) - * 0.001, - POSTEQ_CPU_TIME: sum( - [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - ) - * 0.001, - POSTEQ_CPU_TIME_BACKWARD: sum( - [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + key: sum([rdata[key] for rdata in result[RDATAS]]) * 0.001 + for key in ( + CPU_TIME_TOTAL, + PREEQ_CPU_TIME, + PREEQ_CPU_TIME_BACKWARD, + POSTEQ_CPU_TIME, + POSTEQ_CPU_TIME_BACKWARD, ) - * 0.001, } return values @@ -181,26 +164,14 @@ def _simulation_to_values(self, result, used_time): values = super()._simulation_to_values(result, used_time) # default unit for time in amici is [ms], converted to [s] values |= { - CPU_TIME_TOTAL: sum( - [rdata[CPU_TIME_TOTAL] for rdata in result[RDATAS]] - ) - * 0.001, - PREEQ_CPU_TIME: sum( - [rdata[PREEQ_CPU_TIME] for rdata in result[RDATAS]] - ) - * 0.001, - PREEQ_CPU_TIME_BACKWARD: sum( - [rdata[PREEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] - ) - * 0.001, - POSTEQ_CPU_TIME: sum( - [rdata[POSTEQ_CPU_TIME] for rdata in result[RDATAS]] - ) - * 0.001, - POSTEQ_CPU_TIME_BACKWARD: sum( - [rdata[POSTEQ_CPU_TIME_BACKWARD] for rdata in result[RDATAS]] + key: sum([rdata[key] for rdata in result[RDATAS]]) * 0.001 + for key in ( + CPU_TIME_TOTAL, + PREEQ_CPU_TIME, + PREEQ_CPU_TIME_BACKWARD, + POSTEQ_CPU_TIME, + POSTEQ_CPU_TIME_BACKWARD, ) - * 0.001, } return values