From 51929483d215711865f2205e5eac6c00b7a0e87a Mon Sep 17 00:00:00 2001 From: Chris-Sigopt <92406452+Chris-Sigopt@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:31:59 -0700 Subject: [PATCH] Remove hyperopt (#467) * Remove hyperopt * Oops --- README.md | 3 + integration_test/test_hyperopt.py | 135 ------------------------------ requirements-dev.txt | 1 - setup.py | 4 +- sigopt/hyperopt/__init__.py | 4 - sigopt/hyperopt/base.py | 117 -------------------------- sigopt/hyperopt/compat.py | 24 ------ 7 files changed, 4 insertions(+), 284 deletions(-) delete mode 100644 integration_test/test_hyperopt.py delete mode 100644 sigopt/hyperopt/__init__.py delete mode 100644 sigopt/hyperopt/base.py delete mode 100644 sigopt/hyperopt/compat.py diff --git a/README.md b/README.md index d1875d29..d01845a3 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,9 @@ make vulture-allowlist The `vulture` allowlist file `.vulture_allowlist` can be edit to add/remove allowed no use code/parameters. +## Earlier versions +Earlier versions supported a hyperopt integration. Since hyperopt now seems to be a retired project, we are removing this integration. If you need that integration please select version 8.8.3 + ## Acknowledgments We would like to thank the following people for their contributions: diff --git a/integration_test/test_hyperopt.py b/integration_test/test_hyperopt.py deleted file mode 100644 index 37970a9f..00000000 --- a/integration_test/test_hyperopt.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright © 2022 Intel Corporation -# -# SPDX-License-Identifier: MIT -import time -import uuid - -import hyperopt -import numpy as np -import pytest -from hyperopt import STATUS_FAIL, STATUS_OK, SparkTrials, fmin, hp, tpe -from hyperopt.mongoexp import MongoTrials - -import sigopt -from sigopt.hyperopt import SigOptTrials, upload_trials - - -def objective(p, threshold=1.0, max_sleep_time=0.0): - x, y = p["x"], p["y"] - if max_sleep_time > 0: - time.sleep(np.random.rand() * max_sleep_time) - w = np.random.rand() - if w <= threshold: - return {"loss": x**2 + y**2, "sum": x + y, "status": STATUS_OK} - else: - return {"status": STATUS_FAIL} - - -def objective_random(x): - return objective(x, 0.8) - - -def objective_success(x): - return objective(x, 1.0) - - -def objective_fail(x): - return objective(x, -1.0) - - -class TestHyperopt(object): - def run_fmin(self, online=True, upload=True, objective=objective_success, max_evals=3, wrap=None, **kwargs): - project = "hyperopt-integration-test" - if wrap == "mongo": - trials = MongoTrials("mongo://localhost:27017/foo_db/jobs", exp_key=str(uuid.uuid4())) - elif wrap == "spark": - trials = SparkTrials() - else: - trials = None - trials = SigOptTrials(project=project, online=(online and upload), trials=trials) - try: - best = fmin( - objective, - space={"x": hp.uniform("x", -10, 10), "y": hp.uniform("y", -10, 10)}, - algo=tpe.suggest, - max_evals=max_evals, - trials=trials, - **kwargs - ) - except hyperopt.exceptions.AllTrialsFailed: - best = None - if upload and not online: - trials.upload() - return trials, best - - def test_upload_trials(self): - trials, _ = self.run_fmin(upload=False) - tids = upload_trials("hyperopt-integration-test", trials) - self._verify_uploaded_trials(trials, tids) - - def test_trials_delete_all(self): - trials, _ = self.run_fmin(upload=True) - assert len(trials) > 0 and len(trials) == len(trials.uploaded_tids) - trials.delete_all() - assert len(trials) == 0 and len(trials.uploaded_tids) == 0 - - @pytest.mark.parametrize( - "online, upload, wrap, max_evals", - [ - (False, True, None, 3), - (True, True, None, 3), - (False, False, None, 3), - (True, True, "mongo", 3), - (True, True, "spark", 3), - ], - ) - @pytest.mark.parametrize("objective", [objective_random, objective_success, objective_fail]) - def test_fmin(self, online, upload, objective, wrap, max_evals): - trials, best = self.run_fmin(online, upload, objective, max_evals=max_evals, wrap=wrap) - self._verify_best_trial(best, trials, objective) - self._verify_runs(trials, upload, max_evals) - - def _verify_best_trial(self, best, trials, objective): - if objective is objective_fail: - assert best is None - if objective is objective_success: - assert best is not None - losses = [r["loss"] for r in trials.results if r["status"] == STATUS_OK] - assert ((not best) and (not losses)) or (best and losses) - if best: - loss = objective_success(best)["loss"] - assert loss == min(losses) - - def _verify_runs(self, trials, upload, max_evals): - if not upload: - assert len(trials.uploaded_tids) == 0 - return - assert len(trials.uploaded_tids) == max_evals - self._verify_uploaded_trials(trials, trials.uploaded_tids) - - def _verify_uploaded_trials(self, trials, uploaded_tids): - trial_dict = { - tid: (result, parameters) for tid, result, parameters in zip(trials.tids, trials.results, trials.parameters) - } - - def run_result(run): - result = {} - if run.state == "completed": - result["status"] = STATUS_OK - for m in run.values: - result[m] = run.values[m].value - elif run.state == "failed": - result["status"] = STATUS_FAIL - else: - result["status"] = None - return result - - def run_parameters(run): - return dict(run.assignments) - - run_dict = {} - for tid, run_id in uploaded_tids.items(): - run = sigopt.get_run(run_id) - run_dict[tid] = (run_result(run), run_parameters(run)) - - assert trial_dict == run_dict diff --git a/requirements-dev.txt b/requirements-dev.txt index ef146be3..2e825f48 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,6 @@ numpy>=1.15.0,<2.0.0 Pillow pre-commit>=2.5.2,<3 pylint==2.9.6 -pymongo>=3.12.3,<4 pyspark pytest>=7.2.1 scikit-learn>=1.5.0,<2 diff --git a/setup.py b/setup.py index 95b2b631..ea984637 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ dev_install_requires = requirements_dev_fp.read().split("\n") xgboost_install_requires = ["xgboost>=1.3.1", "numpy>=1.15.0"] -hyperopt_install_requires = ["hyperopt>=0.2.7"] lite_install_requires = ["sigoptlite>=0.1.1"] setup( @@ -50,8 +49,7 @@ }, install_requires=install_requires, extras_require={ - "dev": dev_install_requires + xgboost_install_requires + hyperopt_install_requires + lite_install_requires, - "hyperopt": hyperopt_install_requires, + "dev": dev_install_requires + xgboost_install_requires + lite_install_requires, "xgboost": xgboost_install_requires, "lite": lite_install_requires, }, diff --git a/sigopt/hyperopt/__init__.py b/sigopt/hyperopt/__init__.py deleted file mode 100644 index fe9adc21..00000000 --- a/sigopt/hyperopt/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright © 2022 Intel Corporation -# -# SPDX-License-Identifier: MIT -from .base import SigOptTrials, upload_trials diff --git a/sigopt/hyperopt/base.py b/sigopt/hyperopt/base.py deleted file mode 100644 index 2c85ff33..00000000 --- a/sigopt/hyperopt/base.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright © 2022 Intel Corporation -# -# SPDX-License-Identifier: MIT -from .. import SigOptFactory -from ..defaults import get_default_name -from ..local_run_context import LocalRunContext -from .compat import STATUS_FAIL, STATUS_OK, Trials - - -HYPEROPT_SOURCE_NAME = "Hyperopt" -HYPEROPT_SOURCE_PRIORITY = 1 - - -class SigOptTrials(object): - def __init__(self, project, trials=None, online=True): - self.factory = SigOptFactory(project) - self.online = online - self._trials = trials if trials is not None else Trials() - self.uploaded_tids = {} - - if online: - self.saved_refresh = getattr(self._trials, "refresh") - - def new_refresh(): - r = self.saved_refresh() - self.do_refresh() - return r - - setattr(self._trials, "refresh", new_refresh) - - @property - def parameters(self): - return [self.trial_parameters(trial) for trial in self.trials] - - def upload(self, trials=None, validate=False): - new_trials = [] - trials = trials if trials is not None else self.trials - if validate: - for trial in trials: - self._trials.assert_valid_trial(trial) - for trial in trials: - result = trial["result"] - status = result.get("status") - if status in [STATUS_OK, STATUS_FAIL]: - tid = trial["tid"] - if tid not in self.uploaded_tids: - new_trials.append(trial) - ids = self._upload(new_trials) - self.uploaded_tids.update(ids) - return ids - - def trial_to_run(self, trial): - metadata = {"optimizer": "hyperopt"} - result = trial["result"] - metrics = {k: v for k, v in result.items() if isinstance(v, (int, float))} - parameters = self.trial_parameters(trial) - status = result.get("status") - run = LocalRunContext(name=get_default_name(self.factory.project), metadata=metadata) - run.log_parameters( - parameters, - source=HYPEROPT_SOURCE_NAME, - source_meta={"sort": HYPEROPT_SOURCE_PRIORITY, "default_show": True}, - ) - if status == STATUS_OK: - if not metrics: - raise ValueError("No metrics found in trial result") - run.log_metrics(metrics) - run.log_state("completed") - elif status == STATUS_FAIL: - run.log_failure() - else: - raise ValueError(f"status must be {STATUS_OK} or {STATUS_FAIL}, actually {status}") - return run.get() - - def _upload(self, trials): - runs = [self.trial_to_run(trial) for trial in trials] - runs = self.factory.upload_runs(runs) - return {trial["tid"]: run.id for trial, run in zip(trials, runs)} - - def trial_parameters(self, trial): - vals = trial.get("misc", {}).get("vals", {}) - rval = {} - for k, v in vals.items(): - if v: - rval[k] = v[0] - return rval - - def do_refresh(self): - if self.online: - self.upload() - - def delete_all(self): - self.uploaded_tids.clear() - self._trials.delete_all() - - def __iter__(self): - return iter(self._trials) - - def __len__(self): - return len(self._trials) - - def __getitem__(self, item): - return self._trials[item] - - def __getattr__(self, name): - attrs = dir(self) - if name in attrs: - return attrs[name] - try: - return getattr(self._trials, name) - except AttributeError as e: - raise AttributeError(f"{self.__class__.__name__} object has no attribute {name}") from e - - -def upload_trials(project, trials): - st = SigOptTrials(project=project, online=False) - return st.upload(trials, validate=True) diff --git a/sigopt/hyperopt/compat.py b/sigopt/hyperopt/compat.py deleted file mode 100644 index 5f73454f..00000000 --- a/sigopt/hyperopt/compat.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright © 2022 Intel Corporation -# -# SPDX-License-Identifier: MIT -# pylint: disable=unused-import -from packaging.version import parse - - -MIN_REQUIRED_HYPEROPT_VERSION = "0.2.7" - -try: - import hyperopt - from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin - -except ImportError as ie_xgb: - raise ImportError( - "hyperopt needs to be installed in order to use sigopt.hyperopt.run" - " functionality. Try running `pip install 'sigopt[hyperopt]'`." - ) from ie_xgb - -if parse(hyperopt.__version__) < parse(MIN_REQUIRED_HYPEROPT_VERSION): - raise ImportError( - f"sigopt.hyperopt is compatible with hyperopt>={MIN_REQUIRED_HYPEROPT_VERSION}." - f" You have version {hyperopt.__version__} installed." - )