Refactor/logging (#519)

.tools/envs/testenv-linux.yml

@@ -33,4 +33,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

.tools/envs/testenv-others.yml

@@ -32,4 +32,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

.tools/envs/testenv-pandas.yml

@@ -30,4 +30,5 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
       - -e ../../

CHANGES.md

@@ -29,6 +29,13 @@ the major changes are:
 
 ### Breaking changes
 
+- When providing a path for the argument `logging` of the functions
+  `maximize` and `minimize` and the file already exists, the default
+  behavior is to raise an error now. Replacement or extension
+  of an existing file must be explicitly configured.
+- The argument `if_table_exists` has no effect anymore and a
+  corresponding warning is raised.
+
 
 ### Deprecations
 
@@ -55,7 +62,10 @@ the major changes are:
     - `convergence_scaled_gradient_tolerance` -> `convergence_gtol_scaled`
     - `stopping_max_criterion_evaluations` -> `stopping_maxfun`
     - `stopping_max_iterations` -> `stopping_maxiter`
-
+- The `log_options` argument of `minimize` and `maximize` is deprecated,
+  an according FutureWarning is raised.
+- The class `OptimizeLogReader` is deprecated and redirects to
+  `SQLiteLogReader`.
 
 ## 0.4.7
 

docs/source/how_to/how_to_logging.ipynb

@@ -53,6 +53,32 @@
     ")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In case the SQLite file already exists, this will raise a `FileExistsError` to prevent from accidentally polluting an existing database. If you want to reuse\n",
+    "an existing database on purpose, you must explicitly provide the corresponding option for `if_database_exists`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "log_options = om.SQLiteLogOptions(\n",
+    "    \"my_log.db\", if_database_exists=om.ExistenceStrategy.EXTEND\n",
+    ")\n",
+    "\n",
+    "res = om.minimize(\n",
+    "    fun=sphere,\n",
+    "    params=np.arange(5),\n",
+    "    algorithm=\"scipy_lbfgsb\",\n",
+    "    logging=log_options,\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -63,7 +89,7 @@
     "\n",
     "However, this makes writing logs rather slow, which becomes notable when the criterion function is very fast. \n",
     "\n",
-    "In that case, you can enable ``\"fast_logging\"``, which is still quite safe!"
+    "In that case, you can enable `fast_logging`, which is still quite safe!"
    ]
   },
   {
@@ -72,22 +98,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "log_options = om.SQLiteLogOptions(\"my_log.db\", fast_logging=True)\n",
+    "\n",
     "res = om.minimize(\n",
     "    fun=sphere,\n",
     "    params=np.arange(5),\n",
     "    algorithm=\"scipy_lbfgsb\",\n",
-    "    logging=\"my_log.db\",\n",
-    "    log_options={\"fast_logging\": True},\n",
+    "    logging=log_options,\n",
     ")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Handling existing tables\n",
-    "\n",
-    "By default, we only append to databases and do not overwrite data in them. You have a few options to change this:"
+    "## Reading the log\n",
+    "To read the log after an optimization, extract the logger from the optimization result:"
    ]
   },
   {
@@ -101,18 +127,16 @@
     "    params=np.arange(5),\n",
     "    algorithm=\"scipy_lbfgsb\",\n",
     "    logging=\"my_log.db\",\n",
-    "    log_options={\n",
-    "        \"if_database_exists\": \"replace\",  # one of \"raise\", \"replace\", \"extend\",\n",
-    "        \"if_table_exists\": \"replace\",  # one of \"raise\", \"replace\", \"extend\"\n",
-    "    },\n",
-    ")"
+    ")\n",
+    "\n",
+    "reader = res.logger"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reading the log"
+    "Alternatively, you can create the reader like this:"
    ]
   },
   {
@@ -121,7 +145,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "reader = om.OptimizeLogReader(\"my_log.db\")"
+    "reader = om.SQLiteLogReader(\"my_log.db\")"
    ]
   },
   {

environment.yml

@@ -47,3 +47,4 @@ dependencies:
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests
       - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests

src/estimagic/estimate_ml.py

@@ -60,8 +60,7 @@ def estimate_ml(
     *,
     bounds=None,
     constraints=None,
-    logging=False,
-    log_options=None,
+    logging=None,
     loglike_kwargs=None,
     jacobian=None,
     jacobian_kwargs=None,
@@ -71,6 +70,7 @@ def estimate_ml(
     hessian_numdiff_options=None,
     design_info=None,
     # deprecated
+    log_options=None,
     lower_bounds=None,
     upper_bounds=None,
     numdiff_options=None,

src/estimagic/estimate_msm.py

@@ -67,14 +67,14 @@ def estimate_msm(
     *,
     bounds=None,
     constraints=None,
-    logging=False,
-    log_options=None,
+    logging=None,
     simulate_moments_kwargs=None,
     weights="diagonal",
     jacobian=None,
     jacobian_kwargs=None,
     jacobian_numdiff_options=None,
     # deprecated
+    log_options=None,
     lower_bounds=None,
     upper_bounds=None,
     numdiff_options=None,

src/optimagic/__init__.py

@@ -20,6 +20,15 @@
 )
 from optimagic.differentiation.derivatives import first_derivative, second_derivative
 from optimagic.differentiation.numdiff_options import NumdiffOptions
+from optimagic.logging import (
+    ExistenceStrategy as ExistenceStrategy,
+)
+from optimagic.logging import (
+    SQLiteLogOptions as SQLiteLogOptions,
+)
+from optimagic.logging import (
+    SQLiteLogReader as SQLiteLogReader,
+)
 from optimagic.logging.read_log import OptimizeLogReader
 from optimagic.optimization.fun_value import (
     FunctionValue,

src/optimagic/deprecations.py

@@ -1,10 +1,16 @@
+import logging
 import warnings
 from dataclasses import replace
 from functools import wraps
-from typing import Any, Callable, ParamSpec
+from pathlib import Path
+from typing import Any, Callable, ParamSpec, cast
 
 from optimagic import mark
 from optimagic.constraints import Constraint, InvalidConstraintError
+from optimagic.logging.logger import (
+    LogOptions,
+    SQLiteLogOptions,
+)
 from optimagic.optimization.fun_value import (
     LeastSquaresFunctionValue,
     LikelihoodFunctionValue,
@@ -13,6 +19,8 @@
 from optimagic.parameters.bounds import Bounds
 from optimagic.typing import AggregationLevel
 
+_logger = logging.getLogger(__name__)
+
 
 def throw_criterion_future_warning():
     msg = (
@@ -461,6 +469,48 @@ def replace_and_warn_about_deprecated_derivatives(candidate, name):
     return out
 
 
+def handle_log_options_throw_deprecated_warning(
+    log_options: dict[str, Any], logger: str | Path | LogOptions | None
+) -> str | Path | LogOptions | None:
+    msg = (
+        "Usage of the parameter log_options is deprecated "
+        "and will be removed in a future version. "
+        "Provide a LogOptions instance for the parameter `logging`, if you need to "
+        "configure the logging."
+    )
+    warnings.warn(msg, FutureWarning)
+    logging_is_path_or_string = isinstance(logger, str) or isinstance(logger, Path)
+    log_options_is_dict = isinstance(log_options, dict)
+    compatible_keys = {"fast_logging", "if_table_exists", "if_database_exists"}
+    log_options_is_compatible = set(log_options.keys()).issubset(compatible_keys)
+
+    if logging_is_path_or_string:
+        if log_options_is_dict and log_options_is_compatible:
+            warnings.warn(
+                f"\nUsing {log_options=} to create an instance of SQLiteLogOptions. "
+                f"This mechanism will be removed in the future.",
+                FutureWarning,
+            )
+            if "if_table_exists" in log_options:
+                warnings.warn(
+                    "Found 'if_table_exists' in options dictionary. "
+                    "This option is deprecated and setting it has no effect.",
+                    FutureWarning,
+                )
+                log_options = {
+                    k: v for k, v in log_options.items() if k != "if_table_exists"
+                }
+            return SQLiteLogOptions(cast(str | Path, logger), **log_options)
+        elif not log_options_is_compatible:
+            raise ValueError(
+                f"Found string or path for logger argument, but parameter"
+                f" {log_options=} is not compatible to {compatible_keys=}."
+                f"Explicitly create a Logger instance for configuration."
+            )
+
+    return logger
+
+
 def pre_process_constraints(
     constraints: list[Constraint | dict[str, Any]] | Constraint | dict[str, Any] | None,
 ) -> list[dict[str, Any]]:

src/optimagic/logging/__init__.py

@@ -0,0 +1,7 @@
+from .logger import (
+    SQLiteLogOptions as SQLiteLogOptions,
+)
+from .logger import (
+    SQLiteLogReader as SQLiteLogReader,
+)
+from .types import ExistenceStrategy as ExistenceStrategy