Merge pull request #980 from automl/fix/hyperband_bracket_scaling

Adjust hyperband configuration distribution across brackets

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Callbacks registration is now a public method of the optimizer and allows callbacks to be inserted at a specific position.
 
 ## Bugfixes
+- Adjust amount of configurations in different stages of hyperband brackets to conform to the original paper.
 - Fix validation in smbo to use the seed in the scenario.
 - Change order of callbacks, intensifier callback for incumbent selection is now the first callback. 
 

smac/intensifier/hyperband.py

@@ -2,8 +2,6 @@
 
 from typing import Any
 
-import numpy as np
-
 from smac.intensifier.successive_halving import SuccessiveHalving
 
 
@@ -22,36 +20,22 @@ def __post_init__(self) -> None:
 
         min_budget = self._min_budget
         max_budget = self._max_budget
+        assert min_budget is not None and max_budget is not None
         eta = self._eta
 
         # The only difference we have to do is change max_iterations, n_configs_in_stage, budgets_in_stage
-        s_max = int(np.floor(np.log(max_budget / min_budget) / np.log(eta)))  # type: ignore[operator]
-
-        max_iterations: dict[int, int] = {}
-        n_configs_in_stage: dict[int, list] = {}
-        budgets_in_stage: dict[int, list] = {}
-
-        for i in range(s_max + 1):
-            max_iter = s_max - i
-            n_initial_challengers = int(eta**max_iter)
-
-            # How many configs in each stage
-            linspace = -np.linspace(0, max_iter, max_iter + 1)
-            n_configs_ = n_initial_challengers * np.power(eta, linspace)
-            n_configs = np.array(np.round(n_configs_), dtype=int).tolist()
-
-            # How many budgets in each stage
-            linspace = -np.linspace(max_iter, 0, max_iter + 1)
-            budgets = (max_budget * np.power(eta, linspace)).tolist()
-
-            max_iterations[i] = max_iter + 1
-            n_configs_in_stage[i] = n_configs
-            budgets_in_stage[i] = budgets
-
-        self._s_max = s_max
-        self._max_iterations = max_iterations
-        self._n_configs_in_stage = n_configs_in_stage
-        self._budgets_in_stage = budgets_in_stage
+        self._s_max = self._get_max_iterations(eta, max_budget, min_budget)  # type: ignore[operator]
+        self._max_iterations: dict[int, int] = {}
+        self._n_configs_in_stage: dict[int, list] = {}
+        self._budgets_in_stage: dict[int, list] = {}
+
+        for i in range(self._s_max + 1):
+            max_iter = self._s_max - i
+
+            self._budgets_in_stage[i], self._n_configs_in_stage[i] = self._compute_configs_and_budgets_for_stages(
+                eta, max_budget, max_iter, self._s_max
+            )
+            self._max_iterations[i] = max_iter + 1
 
     def get_state(self) -> dict[str, Any]:  # noqa: D102
         state = super().get_state()

smac/intensifier/successive_halving.py

@@ -2,6 +2,7 @@
 
 from typing import Any, Iterator
 
+import math
 from collections import defaultdict
 
 import numpy as np
@@ -166,17 +167,8 @@ def __post_init__(self) -> None:
         )
 
         # Pre-computing Successive Halving variables
-        max_iter = int(np.floor(np.log(max_budget / min_budget) / np.log(eta)))
-        n_initial_challengers = int(eta**max_iter)
-
-        # How many configs in each stage
-        linspace = -np.linspace(0, max_iter, max_iter + 1)
-        n_configs_ = n_initial_challengers * np.power(eta, linspace)
-        n_configs = np.array(np.round(n_configs_), dtype=int).tolist()
-
-        # How many budgets in each stage
-        linspace = -np.linspace(max_iter, 0, max_iter + 1)
-        budgets = (max_budget * np.power(eta, linspace)).tolist()
+        max_iter = self._get_max_iterations(eta, max_budget, min_budget)
+        budgets, n_configs = self._compute_configs_and_budgets_for_stages(eta, max_budget, max_iter)
 
         # Global variables
         self._min_budget = min_budget
@@ -187,6 +179,30 @@ def __post_init__(self) -> None:
         self._n_configs_in_stage: dict[int, list] = {0: n_configs}
         self._budgets_in_stage: dict[int, list] = {0: budgets}
 
+    @staticmethod
+    def _get_max_iterations(eta: int, max_budget: float | int, min_budget: float | int) -> int:
+        return int(np.floor(np.log(max_budget / min_budget) / np.log(eta)))
+
+    @staticmethod
+    def _compute_configs_and_budgets_for_stages(
+        eta: int, max_budget: float | int, max_iter: int, s_max: int | None = None
+    ) -> tuple[list[int], list[int]]:
+        if s_max is None:
+            s_max = max_iter
+
+        n_initial_challengers = math.ceil((eta**max_iter) * (s_max + 1) / (max_iter + 1))
+
+        # How many configs in each stage
+        lin_space = -np.linspace(0, max_iter, max_iter + 1)
+        n_configs_ = np.floor(n_initial_challengers * np.power(eta, lin_space))
+        n_configs = np.array(np.round(n_configs_), dtype=int).tolist()
+
+        # How many budgets in each stage
+        lin_space = -np.linspace(max_iter, 0, max_iter + 1)
+        budgets = (max_budget * np.power(eta, lin_space)).tolist()
+
+        return budgets, n_configs
+
     def get_state(self) -> dict[str, Any]:  # noqa: D102
         # Replace config by dict
         tracker: dict[str, list[tuple[int | None, list[dict]]]] = defaultdict(list)

tests/test_intensifier/test_hyperband.py

@@ -30,10 +30,10 @@ def test_initialization(make_scenario, configspace_small):
     assert intensifier._max_iterations[4] == 1
 
     assert intensifier._n_configs_in_stage[0] == [81, 27, 9, 3, 1]
-    assert intensifier._n_configs_in_stage[1] == [27, 9, 3, 1]
-    assert intensifier._n_configs_in_stage[2] == [9, 3, 1]
-    assert intensifier._n_configs_in_stage[3] == [3, 1]  # in the paper it's 6 and 2 which is false
-    assert intensifier._n_configs_in_stage[4] == [1]  # in the paper it's 5 which is false?
+    assert intensifier._n_configs_in_stage[1] == [34, 11, 3, 1]
+    assert intensifier._n_configs_in_stage[2] == [15, 5, 1]
+    assert intensifier._n_configs_in_stage[3] == [8, 2]
+    assert intensifier._n_configs_in_stage[4] == [5]
 
     assert intensifier._budgets_in_stage[0] == [1, 3, 9, 27, 81]
     assert intensifier._budgets_in_stage[1] == [3, 9, 27, 81]
@@ -75,7 +75,7 @@ def test_state(make_scenario, configspace_small, make_config_selector):
     gen = iter(intensifier)
 
     # Add some configs to the tracker
-    for _ in range(10):
+    for _ in range(12):
         trial = next(gen)
         runhistory.add_running_trial(trial)  # We have to mark it as running manually
         intensifier.update_incumbents(trial.config)