From daac0cecbd6f75c20ba79241bfa6fdee79893ad8 Mon Sep 17 00:00:00 2001 From: Tim Mensinger Date: Fri, 25 Oct 2024 11:18:49 +0200 Subject: [PATCH 1/5] Support numpy>=2 --- .tools/envs/testenv-linux.yml | 2 +- .tools/envs/testenv-others.yml | 2 +- .tools/envs/testenv-pandas.yml | 2 +- docs/source/how_to/how_to_multistart.ipynb | 2 +- environment.yml | 2 +- pyproject.toml | 2 +- src/optimagic/differentiation/richardson_extrapolation.py | 4 ++-- src/optimagic/optimizers/_pounders/gqtpar.py | 4 ++-- src/optimagic/parameters/kernel_transformations.py | 2 +- .../optimagic/optimization/test_with_nonlinear_constraints.py | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index fa5ece402..f0485262d 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -16,7 +16,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy<2.0 # run, tests + - numpy # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 4467a19b0..59cc38781 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -14,7 +14,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy<2.0 # run, tests + - numpy # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index 757d5f39a..bf66c7ea5 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -15,7 +15,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy<2.0 # run, tests + - numpy # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests - scipy>=1.2.1 # run, tests diff --git a/docs/source/how_to/how_to_multistart.ipynb b/docs/source/how_to/how_to_multistart.ipynb index 6c1589967..73ff8f171 100644 --- a/docs/source/how_to/how_to_multistart.ipynb +++ b/docs/source/how_to/how_to_multistart.ipynb @@ -436,7 +436,7 @@ "metadata": {}, "outputs": [], "source": [ - "np.row_stack(res.multistart_info.exploration_sample).shape" + "np.vstack(res.multistart_info.exploration_sample).shape" ] }, { diff --git a/environment.yml b/environment.yml index d321d992e..1ae4aacde 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - toml # dev - cloudpickle # run, tests - joblib # run, tests - - numpy<2.0 # run, tests + - numpy # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests diff --git a/pyproject.toml b/pyproject.toml index b04c3f06d..fe3c69225 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-python = ">=3.10" dependencies = [ "cloudpickle", "joblib", - "numpy<2.0", + "numpy", "pandas", "plotly", "pybaum>=0.1.2", diff --git a/src/optimagic/differentiation/richardson_extrapolation.py b/src/optimagic/differentiation/richardson_extrapolation.py index fe1842e40..3aa189527 100644 --- a/src/optimagic/differentiation/richardson_extrapolation.py +++ b/src/optimagic/differentiation/richardson_extrapolation.py @@ -272,7 +272,7 @@ def _compute_step_ratio(steps): """ ratios = steps[1:, :] / steps[:-1, :] - ratios = ratios[np.isfinite(ratios)] + finite_ratios = ratios[np.isfinite(ratios)] - step_ratio = ratios.flat[0] + step_ratio = finite_ratios.item(0) return step_ratio diff --git a/src/optimagic/optimizers/_pounders/gqtpar.py b/src/optimagic/optimizers/_pounders/gqtpar.py index a31e59b6a..bf9eb68dd 100644 --- a/src/optimagic/optimizers/_pounders/gqtpar.py +++ b/src/optimagic/optimizers/_pounders/gqtpar.py @@ -75,7 +75,7 @@ def gqtpar(model, x_candidate, *, k_easy=0.1, k_hard=0.2, maxiter=200): zero_threshold = ( model.square_terms.shape[0] * np.finfo(float).eps - * np.linalg.norm(model.square_terms, np.Inf) + * np.linalg.norm(model.square_terms, np.inf) ) stopping_criteria = { "k_easy": k_easy, @@ -175,7 +175,7 @@ def _get_initial_guess_for_lambdas( gradient_norm = np.linalg.norm(main_model.linear_terms) model_hessian = main_model.square_terms - hessian_infinity_norm = np.linalg.norm(model_hessian, np.Inf) + hessian_infinity_norm = np.linalg.norm(model_hessian, np.inf) hessian_frobenius_norm = np.linalg.norm(model_hessian, "fro") hessian_gershgorin_lower, hessian_gershgorin_upper = _compute_gershgorin_bounds( diff --git a/src/optimagic/parameters/kernel_transformations.py b/src/optimagic/parameters/kernel_transformations.py index b4a8540c9..d371b6851 100644 --- a/src/optimagic/parameters/kernel_transformations.py +++ b/src/optimagic/parameters/kernel_transformations.py @@ -496,7 +496,7 @@ def _transformation_matrix(dim): rows = [_unit_vector_or_zeros(i, dim**2) for i in indices] - transformer = np.row_stack(rows) + transformer = np.vstack(rows) return transformer diff --git a/tests/optimagic/optimization/test_with_nonlinear_constraints.py b/tests/optimagic/optimization/test_with_nonlinear_constraints.py index 123cebe86..bc28a2dcf 100644 --- a/tests/optimagic/optimization/test_with_nonlinear_constraints.py +++ b/tests/optimagic/optimization/test_with_nonlinear_constraints.py @@ -41,7 +41,7 @@ def constraint_func(x): return np.array([value - 1, 2 - value]) def constraint_jac(x): - return 2 * np.row_stack((x.reshape(1, -1), -x.reshape(1, -1))) + return 2 * np.vstack((x.reshape(1, -1), -x.reshape(1, -1))) constraints_long = om.NonlinearConstraint( func=constraint_func, From 8fb3eb1b554d39918a810b247c67e01359885b54 Mon Sep 17 00:00:00 2001 From: Tim Mensinger Date: Fri, 25 Oct 2024 11:23:31 +0200 Subject: [PATCH 2/5] Test optimagic with numpy<2 on GHA --- .github/workflows/main.yml | 25 ++++++++++++++++++++++ .tools/envs/testenv-linux.yml | 2 +- .tools/envs/testenv-numpy.yml | 38 ++++++++++++++++++++++++++++++++++ .tools/envs/testenv-others.yml | 2 +- .tools/envs/testenv-pandas.yml | 2 +- .tools/update_envs.py | 9 ++++++-- environment.yml | 2 +- 7 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 .tools/envs/testenv-numpy.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f50f0b740..49d5d1074 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,6 +96,31 @@ jobs: run: | micromamba activate optimagic pytest -m "not slow and not jax" + run-tests-with-old-numpy: + # This job is only for testing if optimagic works with numpy<2. + name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with numpy 1 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + python-version: + - '3.10' + steps: + - uses: actions/checkout@v4 + - name: create build environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./.tools/envs/testenv-numpy.yml + cache-environment: true + create-args: | + python=${{ matrix.python-version }} + - name: run pytest + shell: bash -l {0} + run: | + micromamba activate optimagic + pytest -m "not slow and not jax" code-in-docs: name: Run code snippets in documentation runs-on: ubuntu-latest diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index f0485262d..dad1f9215 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -16,7 +16,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy # run, tests + - numpy >= 2 # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml new file mode 100644 index 000000000..0a1d55176 --- /dev/null +++ b/.tools/envs/testenv-numpy.yml @@ -0,0 +1,38 @@ +--- +name: optimagic +channels: + - conda-forge + - nodefaults +dependencies: + - numpy<2 + - cyipopt>=1.4.0 # dev, tests + - pygmo>=2.19.0 # dev, tests + - nlopt # dev, tests + - pip # dev, tests, docs + - pytest # dev, tests + - pytest-cov # tests + - pytest-xdist # dev, tests + - statsmodels # dev, tests + - cloudpickle # run, tests + - joblib # run, tests + - pandas # run, tests + - plotly # run, tests + - pybaum >= 0.1.2 # run, tests + - scipy>=1.2.1 # run, tests + - sqlalchemy # run, tests + - seaborn # dev, tests + - mypy>=1.11 # dev, tests + - pyyaml # dev, tests + - jinja2 # dev, tests + - annotated-types # dev, tests + - pip: # dev, tests, docs + - DFO-LS # dev, tests + - Py-BOBYQA # dev, tests + - fides==0.7.4 # dev, tests + - kaleido # dev, tests + - pandas-stubs # dev, tests + - types-cffi # dev, tests + - types-openpyxl # dev, tests + - types-jinja2 # dev, tests + - sqlalchemy-stubs # dev, tests + - -e ../../ diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 59cc38781..93849f10a 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -14,7 +14,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy # run, tests + - numpy >= 2 # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index bf66c7ea5..05f005c3d 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -15,7 +15,7 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy # run, tests + - numpy >= 2 # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests - scipy>=1.2.1 # run, tests diff --git a/.tools/update_envs.py b/.tools/update_envs.py index 0c773a5b8..e74fe8fc9 100644 --- a/.tools/update_envs.py +++ b/.tools/update_envs.py @@ -38,6 +38,11 @@ def main(): test_env_pandas = [line for line in test_env_pandas if "pandas" not in line] test_env_pandas.insert(_insert_idx, " - pandas<2.0.0") + ## test environment for numpy version 1 + test_env_numpy = deepcopy(test_env) + test_env_numpy = [line for line in test_env_numpy if "numpy" not in line] + test_env_numpy.insert(_insert_idx, " - numpy<2") + # create docs testing environment docs_env = [line for line in lines if _keep_line(line, "docs")] @@ -45,8 +50,8 @@ def main(): # write environments for name, env in zip( - ["linux", "others", "pandas"], - [test_env_linux, test_env_others, test_env_pandas], + ["linux", "others", "pandas", "numpy"], + [test_env_linux, test_env_others, test_env_pandas, test_env_numpy], strict=False, ): # Specify newline to avoid wrong line endings on Windows. diff --git a/environment.yml b/environment.yml index 1ae4aacde..88882e7aa 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - toml # dev - cloudpickle # run, tests - joblib # run, tests - - numpy # run, tests + - numpy >= 2 # run, tests - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests From 3985f33e96ecf78a1b53db68b03e7ef0a777fb37 Mon Sep 17 00:00:00 2001 From: Tim Mensinger Date: Fri, 25 Oct 2024 11:27:18 +0200 Subject: [PATCH 3/5] Pin mypy version in environment to that of pre-commit hooks --- .tools/envs/testenv-linux.yml | 2 +- .tools/envs/testenv-numpy.yml | 2 +- .tools/envs/testenv-others.yml | 2 +- .tools/envs/testenv-pandas.yml | 2 +- environment.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index dad1f9215..b2bcaac00 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -23,7 +23,7 @@ dependencies: - scipy>=1.2.1 # run, tests - sqlalchemy # run, tests - seaborn # dev, tests - - mypy>=1.11 # dev, tests + - mypy=1.11 # dev, tests - pyyaml # dev, tests - jinja2 # dev, tests - annotated-types # dev, tests diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index 0a1d55176..0241dc84e 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -21,7 +21,7 @@ dependencies: - scipy>=1.2.1 # run, tests - sqlalchemy # run, tests - seaborn # dev, tests - - mypy>=1.11 # dev, tests + - mypy=1.11 # dev, tests - pyyaml # dev, tests - jinja2 # dev, tests - annotated-types # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 93849f10a..0998cf32f 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -21,7 +21,7 @@ dependencies: - scipy>=1.2.1 # run, tests - sqlalchemy # run, tests - seaborn # dev, tests - - mypy>=1.11 # dev, tests + - mypy=1.11 # dev, tests - pyyaml # dev, tests - jinja2 # dev, tests - annotated-types # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index 05f005c3d..48406a95f 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -21,7 +21,7 @@ dependencies: - scipy>=1.2.1 # run, tests - sqlalchemy # run, tests - seaborn # dev, tests - - mypy>=1.11 # dev, tests + - mypy=1.11 # dev, tests - pyyaml # dev, tests - jinja2 # dev, tests - annotated-types # dev, tests diff --git a/environment.yml b/environment.yml index 88882e7aa..06e0d88f8 100644 --- a/environment.yml +++ b/environment.yml @@ -32,7 +32,7 @@ dependencies: - sphinx-panels # docs - sphinxcontrib-bibtex # docs - seaborn # dev, tests - - mypy>=1.11 # dev, tests + - mypy=1.11 # dev, tests - pyyaml # dev, tests - jinja2 # dev, tests - furo # dev, docs From b4390196f737eb6be10169f2f2183ff23a78945c Mon Sep 17 00:00:00 2001 From: Tim Mensinger Date: Mon, 28 Oct 2024 09:56:22 +0100 Subject: [PATCH 4/5] Remove numpy<2 restriction from mypy hook --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b00b2f9a7..f5a8c66a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -124,7 +124,7 @@ repos: - id: mypy files: src|tests additional_dependencies: - - numpy<2.0 + - numpy - packaging - pandas-stubs - sqlalchemy-stubs From b7fabd7bd30b9979c8aa0af3bf88ff2207abfbd5 Mon Sep 17 00:00:00 2001 From: Tim Mensinger Date: Mon, 28 Oct 2024 10:08:03 +0100 Subject: [PATCH 5/5] Update testing environments --- .github/workflows/main.yml | 10 ++++++---- .tools/envs/testenv-numpy.yml | 3 +-- .tools/envs/testenv-pandas.yml | 4 ++-- .tools/update_envs.py | 16 +++++++++------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49d5d1074..dd05b6a1f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,9 +70,10 @@ jobs: micromamba activate optimagic pytest -m "not slow and not jax" run-tests-with-old-pandas: - # This job is only for testing if optimagic works with older pandas versions, as - # many pandas functions we use will be deprecated in pandas 3. optimagic's behavior - # for older verions is handled in src/optimagic/compat.py. + # This job is only for testing if optimagic works with pandas<2, as many pandas + # functions we use will be deprecated in pandas 3. optimagic's behavior for older + # verions is handled in src/optimagic/compat.py. For compatibility with we have to + # restrict numpy<2. name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with pandas 1 runs-on: ${{ matrix.os }} strategy: @@ -97,7 +98,8 @@ jobs: micromamba activate optimagic pytest -m "not slow and not jax" run-tests-with-old-numpy: - # This job is only for testing if optimagic works with numpy<2. + # This job is only for testing if optimagic works with numpy<2. Because we already + # test pandas<2 with numpy<2, in this environment we restrict pandas>=2. name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with numpy 1 runs-on: ${{ matrix.os }} strategy: diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index 85329f559..f54e96c17 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -4,6 +4,7 @@ channels: - conda-forge - nodefaults dependencies: + - pandas>=2 - numpy<2 - cyipopt>=1.4.0 # dev, tests - pygmo>=2.19.0 # dev, tests @@ -15,7 +16,6 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - pandas # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests - scipy>=1.2.1 # run, tests @@ -30,7 +30,6 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - kaleido # dev, tests - - pandas-stubs # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests - types-jinja2 # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index e9c2a61d9..6918bafae 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -4,7 +4,8 @@ channels: - conda-forge - nodefaults dependencies: - - pandas<2.0.0 + - pandas<2 + - numpy<2 - cyipopt>=1.4.0 # dev, tests - pygmo>=2.19.0 # dev, tests - nlopt # dev, tests @@ -15,7 +16,6 @@ dependencies: - statsmodels # dev, tests - cloudpickle # run, tests - joblib # run, tests - - numpy >= 2 # run, tests - plotly # run, tests - pybaum >= 0.1.2 # run, tests - scipy>=1.2.1 # run, tests diff --git a/.tools/update_envs.py b/.tools/update_envs.py index e74fe8fc9..a62a44e3d 100644 --- a/.tools/update_envs.py +++ b/.tools/update_envs.py @@ -33,18 +33,20 @@ def main(): ## test environment others test_env_others = deepcopy(test_env) - ## test environment for pandas version 1 + ## test environment for pandas version < 2 (requires numpy < 2) test_env_pandas = deepcopy(test_env) - test_env_pandas = [line for line in test_env_pandas if "pandas" not in line] - test_env_pandas.insert(_insert_idx, " - pandas<2.0.0") + for pkg in ["numpy", "pandas"]: + test_env_pandas = [line for line in test_env_pandas if pkg not in line] + test_env_pandas.insert(_insert_idx, f" - {pkg}<2") - ## test environment for numpy version 1 + ## test environment for numpy version < 2 (with pandas >= 2) test_env_numpy = deepcopy(test_env) - test_env_numpy = [line for line in test_env_numpy if "numpy" not in line] + for pkg in ["numpy", "pandas"]: + test_env_numpy = [line for line in test_env_numpy if pkg not in line] test_env_numpy.insert(_insert_idx, " - numpy<2") + test_env_numpy.insert(_insert_idx, " - pandas>=2") - # create docs testing environment - + # test environment for documentation docs_env = [line for line in lines if _keep_line(line, "docs")] docs_env.append(" - -e ../../") # add local installation