Skip to content

Commit

Permalink
mapping happens outside of runner (#296)
Browse files Browse the repository at this point in the history 
* mapping happens outside of runner

* notebook fixes

* fixed more and cleaned up a little

* remove type ignores
  • Loading branch information
@bertiqwerty
bertiqwerty authored Oct 5, 2023
1 parent 90428d2 commit 669fa44
Show file tree
Hide file tree
Showing 11 changed files with 191 additions and 237 deletions.
12 changes: 9 additions & 3 deletions bofire/runners/hyperoptimize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pandas as pd

import bofire.strategies.api as strategies
import bofire.strategies.mapper as strategy_mapper
from bofire.benchmarks.api import Hyperopt
from bofire.data_models.domain.api import Domain
from bofire.data_models.enum import RegressionMetricsEnum
Expand Down Expand Up @@ -55,11 +56,16 @@ def sample(domain):
strategy.ask(candidate_count=None), return_complete=True
)
else:
strategy_data = (
RandomStrategy
if surrogate_data.hyperconfig.hyperstrategy == "RandomStrategy"
else SoboStrategy
)
experiments = run(
benchmark=benchmark,
strategy_factory=RandomStrategy
if surrogate_data.hyperconfig.hyperstrategy == "RandomStrategy" # type: ignore
else SoboStrategy, # type: ignore
strategy_factory=lambda domain: strategy_mapper.map(
data_model=strategy_data(domain=domain)
),
metric=best,
n_runs=1,
n_iterations=surrogate_data.hyperconfig.n_iterations # type: ignore
Expand Down
18 changes: 7 additions & 11 deletions bofire/runners/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@
from multiprocess.pool import Pool
from tqdm import tqdm

import bofire.strategies.api as strategies
from bofire.benchmarks.api import Benchmark
from bofire.data_models.domain.api import Domain
from bofire.data_models.strategies.api import AnyStrategy
from bofire.strategies.api import Strategy


class StrategyFactory(Protocol):
def __call__(self, domain: Domain) -> AnyStrategy:
def __call__(self, domain: Domain) -> Strategy:
...


Expand Down Expand Up @@ -51,19 +50,16 @@ def autosafe_results(benchmark):
with open(filename, "w") as file:
json.dump(parsed_domain, file)

# sample initial values
strategy = strategy_factory(domain=benchmark.domain)
# sample initial values and tell
if initial_sampler is not None:
if isinstance(initial_sampler, Callable):
X = initial_sampler(benchmark.domain)
XY = benchmark.f(X, return_complete=True)
else:
XY = initial_sampler
strategy_data = strategy_factory(domain=benchmark.domain)
# map it
strategy = strategies.map(strategy_data) # type: ignore
# tell it
if initial_sampler is not None:
strategy.tell(XY) # type: ignore
strategy.tell(XY)

metric_values = np.zeros(n_iterations)
pbar = tqdm(range(n_iterations), position=run_idx)
for i in pbar:
Expand All @@ -80,7 +76,7 @@ def autosafe_results(benchmark):
)
if (i + 1) % safe_intervall == 0:
autosafe_results(benchmark=benchmark)
return strategy.experiments, pd.Series(metric_values) # type: ignore
return strategy.experiments, pd.Series(metric_values)


def run(
Expand Down
5 changes: 4 additions & 1 deletion tests/bofire/runners/test_run.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import pandas as pd

import bofire.strategies.api as strategies
import bofire.strategies.mapper as strategy_mapper
from bofire.benchmarks.multi import ZDT1
from bofire.data_models.domain.api import Domain
from bofire.data_models.strategies.api import (
Expand Down Expand Up @@ -33,7 +34,9 @@ def hypervolume(domain: Domain, experiments: pd.DataFrame) -> float:

results = run(
zdt1,
strategy_factory=qparego_factory,
strategy_factory=lambda domain: strategy_mapper.map(
qparego_factory(domain=domain)
),
n_iterations=n_iterations,
metric=hypervolume,
initial_sampler=sample,
Expand Down
18 changes: 11 additions & 7 deletions tutorials/benchmarks/001-Himmelblau.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -36,14 +36,14 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"run 00 with current best 12.587: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:00<00:00, 57.01it/s]\n"
"run 00 with current best 4.352: 100%|██████████████████████████████████████████████████| 50/50 [00:01<00:00, 28.18it/s]\n"
]
}
],
Expand All @@ -59,7 +59,7 @@
"\n",
"random_results = run(\n",
" Himmelblau(),\n",
" strategy_factory=RandomStrategy,\n",
" strategy_factory=lambda domain: strategies.map(RandomStrategy(domain=domain)),\n",
" n_iterations=50 if not SMOKE_TEST else 1,\n",
" metric=best,\n",
" initial_sampler=sample,\n",
Expand All @@ -84,14 +84,18 @@
"name": "stderr",
"output_type": "stream",
"text": [
"run 00 with current best 0.017: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [06:22<00:00, 7.65s/it]\n"
"run 00 with current best 0.003: 100%|██████████████████████████████████████████████████| 50/50 [07:19<00:00, 8.79s/it]\n"
]
}
],
"source": [
"def strategy_factory(domain: Domain):\n",
" data_model = SoboStrategy(domain=domain, acquisition_function=qEI())\n",
" return strategies.map(data_model)\n",
"\n",
"bo_results = run(\n",
" Himmelblau(),\n",
" strategy_factory=partial(SoboStrategy, acquisition_function=qEI()),\n",
" strategy_factory=strategy_factory,\n",
" n_iterations=50 if not SMOKE_TEST else 1,\n",
" metric=best,\n",
" initial_sampler=sample,\n",
Expand Down Expand Up @@ -121,5 +125,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
102 changes: 15 additions & 87 deletions tutorials/benchmarks/002-DTLZ2.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@
"\n",
"random_results = run(\n",
" DTLZ2(dim=6),\n",
" strategy_factory=RandomStrategy,\n",
" strategy_factory=lambda domain: strategies.map(RandomStrategy(domain=domain)),\n",
" n_iterations=50 if not SMOKE_TEST else 1,\n",
" metric=hypervolume,\n",
" initial_sampler=sample,\n",
Expand All @@ -116,21 +116,25 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"run 00 with current best 0.381: 100%|██████████| 50/50 [02:04<00:00, 2.50s/it]\n"
"run 00 with current best 0.306: 34%|█████████████████ | 17/50 [03:21<05:55, 10.77s/it]"
]
}
],
"source": [
"def strategy_factory(domain: Domain):\n",
" data_model = QehviStrategy(domain=domain, ref_point={\"f_0\": 1.1, \"f_1\": 1.1})\n",
" return strategies.map(data_model)\n",
"\n",
"results = run(\n",
" DTLZ2(dim=6),\n",
" strategy_factory=partial(QehviStrategy, ref_point={\"f_0\": 1.1, \"f_1\": 1.1}),\n",
" strategy_factory=strategy_factory,\n",
" n_iterations=50 if not SMOKE_TEST else 1,\n",
" metric=hypervolume,\n",
" initial_sampler=sample,\n",
Expand All @@ -151,76 +155,9 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>x_0</th>\n",
" <th>x_1</th>\n",
" <th>x_2</th>\n",
" <th>x_3</th>\n",
" <th>x_4</th>\n",
" <th>x_5</th>\n",
" <th>f_0_pred</th>\n",
" <th>f_1_pred</th>\n",
" <th>f_0_sd</th>\n",
" <th>f_1_sd</th>\n",
" <th>f_0_des</th>\n",
" <th>f_1_des</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.400517</td>\n",
" <td>0.417488</td>\n",
" <td>0.573703</td>\n",
" <td>0.319068</td>\n",
" <td>0.746549</td>\n",
" <td>0.225116</td>\n",
" <td>0.772984</td>\n",
" <td>0.802233</td>\n",
" <td>0.269332</td>\n",
" <td>0.295097</td>\n",
" <td>-0.772984</td>\n",
" <td>-0.802233</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" x_0 x_1 x_2 x_3 x_4 x_5 f_0_pred \\\n",
"0 0.400517 0.417488 0.573703 0.319068 0.746549 0.225116 0.772984 \n",
"\n",
" f_1_pred f_0_sd f_1_sd f_0_des f_1_des \n",
"0 0.802233 0.269332 0.295097 -0.772984 -0.802233 "
]
},
"metadata": {},
"output_type": "display_data"
}
],
"outputs": [],
"source": [
"# we get the domain from the benchmark module, in real use case we have to build it on our own\n",
"# make sure that the objective is set correctly\n",
Expand Down Expand Up @@ -358,21 +295,13 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"run 00 with current best 0.360: 100%|██████████| 50/50 [02:58<00:00, 3.57s/it]\n"
]
}
],
"outputs": [],
"source": [
"results_qparego = run(\n",
" DTLZ2(dim=6),\n",
" strategy_factory=partial(QparegoStrategy),\n",
" strategy_factory=lambda domain: strategies.map(QparegoStrategy(domain=domain)),\n",
" n_iterations=50 if not SMOKE_TEST else 1,\n",
" metric=hypervolume,\n",
" initial_sampler=sample,\n",
Expand Down Expand Up @@ -433,7 +362,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "bofire",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -449,13 +378,12 @@
"pygments_lexer": "ipython3",
"version": "3.10.9"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "6f21737eef49beedf03d74399b47fe38d73eff760737ca33d38b9fe616638e91"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
Loading

0 comments on commit 669fa44

Please sign in to comment.