Skip to content

Commit

Permalink
FIX-#7355: Cpu count would be set incorrectly on a cluster (#7356)
Browse files Browse the repository at this point in the history
Signed-off-by: arunjose696 <arunjose696@gmail.com>
  • Loading branch information
arunjose696 authored Aug 1, 2024
1 parent 22ed4d8 commit 6dce30e
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 0 deletions.
18 changes: 18 additions & 0 deletions modin/config/envvars.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,24 @@ class CpuCount(EnvironmentVariable, type=int):

varname = "MODIN_CPUS"

@classmethod
def _put(cls, value: int) -> None:
"""
Put specific value if CpuCount wasn't set by a user yet.
Parameters
----------
value : int
Config value to set.
Notes
-----
This method is used to set CpuCount from cluster resources internally
and should not be called by a user.
"""
if cls.get_value_source() == ValueSource.DEFAULT:
cls.put(value)

@classmethod
def _get_default(cls) -> int:
"""
Expand Down
1 change: 1 addition & 0 deletions modin/core/execution/dask/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,4 @@ def _disable_warnings():

num_cpus = len(client.ncores())
NPartitions._put(num_cpus)
CpuCount._put(num_cpus)
1 change: 1 addition & 0 deletions modin/core/execution/ray/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ def initialize_ray(

num_cpus = int(ray.cluster_resources()["CPU"])
NPartitions._put(num_cpus)
CpuCount._put(num_cpus)

# TODO(https://github.com/ray-project/ray/issues/28216): remove this
# workaround once Ray gives a better way to suppress task errors.
Expand Down
1 change: 1 addition & 0 deletions modin/core/execution/unidist/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def initialize_unidist():

num_cpus = sum(v["CPU"] for v in unidist.cluster_resources().values())
modin_cfg.NPartitions._put(num_cpus)
modin_cfg.CpuCount._put(num_cpus)


def deserialize(obj): # pragma: no cover
Expand Down

0 comments on commit 6dce30e

Please sign in to comment.