Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: make multi-processing in baseline models more consistent #909

Merged
merged 9 commits into from
Jul 12, 2024
Merged

fix: make multi-processing in baseline models more consistent #909

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
adebadf
set context of ProcessPoolExecutors to spawn
sibre28 Jul 12, 2024
8909b31
add imports
sibre28 Jul 12, 2024
c239160
style: apply automated linter fixes
megalinter-bot Jul 12, 2024
298374d
style: apply automated linter fixes
megalinter-bot Jul 12, 2024
320942d
docstring format
sibre28 Jul 12, 2024
dd6db33
Merge remote-tracking branch 'origin/907-multiprocessing-tests-of-bas…
sibre28 Jul 12, 2024
0a48763
Apply suggestions from code review
sibre28 Jul 12, 2024
116ce21
style: apply automated linter fixes
megalinter-bot Jul 12, 2024
7e40c7b
Merge branch 'main' into 907-multiprocessing-tests-of-baseline-models…
sibre28 Jul 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/safeds/ml/classical/classification/_baseline_classifier.py
View file
Open in desktop
sibre28 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from concurrent.futures import ALL_COMPLETED, wait
from typing import Self

from joblib._multiprocessing_helpers import mp
sibre28 marked this conversation as resolved.
Show resolved Hide resolved

from safeds._validation._check_columns_are_numeric import _check_columns_are_numeric
from safeds.data.labeled.containers import TabularDataset
from safeds.exceptions import (
Expand Down Expand Up @@ -86,7 +88,10 @@ def fit(self, train_data: TabularDataset) -> Self:

copied_model = copy.deepcopy(self)

with ProcessPoolExecutor(max_workers=len(self._list_of_model_types)) as executor:
with ProcessPoolExecutor(
max_workers=len(self._list_of_model_types),
mp_context=mp.get_context("spawn"),
) as executor:
futures = []
for model in self._list_of_model_types:
futures.append(executor.submit(_fit_single_model, model, train_data))
Expand Down Expand Up @@ -149,7 +154,10 @@ def predict(self, test_data: TabularDataset) -> dict[str, float]:
raise DatasetMissesDataError
_check_columns_are_numeric(test_data_as_table, test_data.features.add_columns(test_data.target).column_names)

with ProcessPoolExecutor(max_workers=len(self._list_of_model_types)) as executor:
with ProcessPoolExecutor(
max_workers=len(self._list_of_model_types),
mp_context=mp.get_context("spawn"),
) as executor:
results = []
futures = []
for model in self._fitted_models:
Expand Down
12 changes: 10 additions & 2 deletions src/safeds/ml/classical/regression/_baseline_regressor.py
View file
Open in desktop
sibre28 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from concurrent.futures import ALL_COMPLETED, wait
from typing import Self

from joblib._multiprocessing_helpers import mp
sibre28 marked this conversation as resolved.
Show resolved Hide resolved

from safeds._validation._check_columns_are_numeric import _check_columns_are_numeric
from safeds.data.labeled.containers import TabularDataset
from safeds.exceptions import (
Expand Down Expand Up @@ -95,7 +97,10 @@ def fit(self, train_data: TabularDataset) -> Self:

copied_model = copy.deepcopy(self)

with ProcessPoolExecutor(max_workers=len(self._list_of_model_types)) as executor:
with ProcessPoolExecutor(
max_workers=len(self._list_of_model_types),
mp_context=mp.get_context("spawn"),
) as executor:
futures = []
for model in self._list_of_model_types:
futures.append(executor.submit(_fit_single_model, model, train_data))
Expand Down Expand Up @@ -159,7 +164,10 @@ def predict(self, test_data: TabularDataset) -> dict[str, float]:
_check_columns_are_numeric(test_data_as_table, test_data.features.add_columns(test_data.target).column_names)

# Start Processes
with ProcessPoolExecutor(max_workers=len(self._list_of_model_types)) as executor:
with ProcessPoolExecutor(
max_workers=len(self._list_of_model_types),
mp_context=mp.get_context("spawn"),
) as executor:
results = []
futures = []
for model in self._fitted_models:
Expand Down