Minor updates

CHANGELOG.md

@@ -1,8 +1,17 @@
-## [v0.0.5] - [Unreleased]
+## [v0.0.5] - [01/05/2022]
 
 ### Added
-- Adds `SuccessiveHalvingSearch`, `HyperbandSearch`, `PBTSearch` and examples/NB documentation (see issues #3, #4).
+
 - Adds possibility to store and reload entire strategies as pkl file (as asked for in issue #2).
+- Adds `improvement` method indicating if score is better than best stored one
+- Adds save option for best plot
+- Adds `args, kwargs` into decorator
+- Adds synchronous Successive Halving (`SuccessiveHalvingSearch` - issue #3)
+- Adds synchronous HyperBand (`HyperbandSearch` - issue #3)
+- Adds synchronous PBT (`PBTSearch` - issue #4)
+- Adds option to save log in `tell` method
+- Adds small torch mlp example for SH/Hyperband/PBT w. logging/scheduler
+- Adds print welcome/update message for strategy specific info
 
 ### Changed
 - Major internal restructuring:
@@ -11,8 +20,12 @@
   - `log_search`: Add search specific log data to evaluation log
   - `update_search`: Refine search space/change active strategy etc.
 - Also allow to store checkpoint of trained models in `tell` method.
-
-### Fixed
+- Fix logging message when log is stored
+- Make json serializer more robust for numpy data types
+- Robust type checking with `isinstance(self.log[0]["objective"], (float, int, np.integer, np.float))`
+- Update NB to include `mle-scheduler` example
+- Make PBT explore robust for integer/categorical valued hyperparams
+- Calculate total batches & their sizes for hyperband
 
 ## [v0.0.4] - [12/10/2021]
 

README.md

@@ -89,7 +89,7 @@ strategy = GridSearch(
                             "end": 5,
                             "bins": 1}},
     categorical={"arch": ["mlp", "cnn"]},
-    fixed_params={"momentum": 0.9})
+    fixed_params={"momentum": 0.9})  # Add fixed param setting to each config
 
 configs = strategy.ask()
 ```
@@ -107,7 +107,7 @@ strategy = HyperbandSearch(
     categorical={"arch": ["mlp", "cnn"]},
     search_config={"max_resource": 81,
                    "eta": 3},
-    seed_id=42,
+    seed_id=42,  # Fix randomness for reproducibility
     verbose=True)
 
 configs = strategy.ask()
@@ -126,7 +126,7 @@ strategy = PBTSearch(
         "steps_until_ready": 4,
         "num_workers": 10,
     },
-    maximize_objective=True
+    maximize_objective=True  # Max score instead of min
 )
 
 configs = strategy.ask()
@@ -172,6 +172,20 @@ strategy.ask(2, store=True)
 strategy.ask(2, store=True, config_fnames=["conf_0.yaml", "conf_1.yaml"])
 ```
 
+### Storing Checkpoint Paths 🛥️
+
+
+```python
+# Ask for 5 configurations to evaluate and get their scores
+configs = strategy.ask(5)
+values = ...
+# Get list of checkpoint paths corresponding to config runs
+ckpts = [f"ckpt_{i}.pt" for i in range(len(configs))]
+# `tell` parameter configs, eval scores & ckpt paths
+# Required for Halving, Hyperband and PBT
+strategy.tell(configs, scores, ckpts)
+```
+
 ### Retrieving Top Performers & Visualizing Results 📉
 
 ```python
@@ -204,7 +218,7 @@ strategy.tell(...)
 strategy.refine(top_k=2)
 ```
 
-Note that the search space refinement is only implemented for random, SMBO and nevergrad-based search strategies.
+Note that the search space refinement is only implemented for random, SMBO and `nevergrad`-based search strategies.
 
 
 ### Citing the MLE-Infrastructure ✏️
@@ -220,6 +234,6 @@ If you use `mle-hyperopt` in your research, please cite it as follows:
 }
 ```
 
-## Development
+## Development 👷
 
-You can run the test suite via `python -m pytest -vv tests/`. If you find a bug or are missing your favourite feature, feel free to contact me [@RobertTLange](https://twitter.com/RobertTLange) or create an issue :hugs:.
+You can run the test suite via `python -m pytest -vv tests/`. If you find a bug or are missing your favourite feature, feel free to create an issue and/or start [contributing](CONTRIBUTING.md) :hugs:.

examples/run_hb_mlp.py

@@ -5,7 +5,7 @@
 def main():
     """Run Hyperband"""
     strategy = HyperbandSearch(
-        real={"lrate": {"begin": 1e-08, "end": 1e-06, "prior": "uniform"}},
+        real={"lrate": {"begin": 1e-04, "end": 1e-02, "prior": "uniform"}},
         search_config={"max_resource": 27, "eta": 3},
         seed_id=42,
         verbose=True,

examples/run_pbt_mlp.py

@@ -5,7 +5,7 @@
 def main():
     """Run Population-Based Training"""
     strategy = PBTSearch(
-        real={"lrate": {"begin": 1e-08, "end": 1e-06, "prior": "uniform"}},
+        real={"lrate": {"begin": 1e-04, "end": 1e-02, "prior": "uniform"}},
         search_config={
             "exploit": {"strategy": "truncation", "selection_percent": 0.2},
             "explore": {"strategy": "perturbation", "perturb_coeffs": [0.8, 1.2]},

examples/run_sh_mlp.py

@@ -5,7 +5,7 @@
 def main():
     """Run Successive Halving"""
     strategy = HalvingSearch(
-        real={"lrate": {"begin": 1e-08, "end": 1e-06, "prior": "uniform"}},
+        real={"lrate": {"begin": 1e-04, "end": 1e-02, "prior": "uniform"}},
         search_config={"min_budget": 1, "num_arms": 20, "halving_coeff": 2},
         seed_id=42,
         verbose=True,

examples/train_mlp.py

@@ -30,7 +30,7 @@ def main(experiment_dir: str, config_fname: str, seed_id: int):
     elif "pbt_ckpt" in train_config["extra"].keys():
         model.load_state_dict(torch.load(train_config.extra.pbt_ckpt))
 
-    loss_fn = torch.nn.MSELoss(reduction="sum")
+    loss_fn = torch.nn.MSELoss()
 
     if "sh_num_add_iters" in train_config.extra.keys():
         num_gd_steps = train_config.extra.sh_num_add_iters * 50

mle_hyperopt/strategy.py

@@ -119,6 +119,10 @@ def tell(
         if type(ckpt_paths) == str:
             ckpt_paths = [ckpt_paths]
 
+        # Check that checkpoints are provided if using iterative search methods
+        if self.search_name in ["Halving", "Hyperband", "PBT"]:
+            assert ckpt_paths is not None
+
         log_data, clean_prop, clean_perf, clean_ckpt = self.clean_data(
             batch_proposals, perf_measures, ckpt_paths
         )
@@ -233,7 +237,7 @@ def save(self, save_path: str = "search_log.yaml"):
             raise ValueError("Only YAML, JSON or PKL file paths supported.")
         if self.verbose:
             Console().log(
-                f"Stored {self.eval_counter} search iterations --> {save_path}."
+                f"Stored {self.eval_counter} search iterations ➞ {save_path}."
             )
 
     def load(