forked from modin-project/modin
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b608ea0
commit c4e583b
Showing
3 changed files
with
79 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,62 +1,63 @@ | ||
| # Licensed to Modin Development Team under one or more contributor license agreements. | ||
| # See the NOTICE file distributed with this work for additional information regarding | ||
| # copyright ownership. The Modin Development Team licenses this file to you under the | ||
| # Apache License, Version 2.0 (the "License"); you may not use this file except in | ||
| # compliance with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software distributed under | ||
| # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF | ||
| # ANY KIND, either express or implied. See the License for the specific language | ||
| # governing permissions and limitations under the License. | ||
|
|
||
| import unittest.mock as mock | ||
|
|
||
| import pytest | ||
|
|
||
| import modin.pandas as pd | ||
| from modin.config import Engine | ||
|
|
||
| engine = Engine.get() | ||
|
|
||
| # We have to explicitly mock subclass implementations of wait_partitions. | ||
| if engine == "Ray": | ||
| wait_method = ( | ||
| "modin.core.execution.ray.implementations." | ||
| + "pandas_on_ray.partitioning." | ||
| + "PandasOnRayDataframePartitionManager.wait_partitions" | ||
| ) | ||
| elif engine == "Dask": | ||
| wait_method = ( | ||
| "modin.core.execution.dask.implementations." | ||
| + "pandas_on_dask.partitioning." | ||
| + "PandasOnDaskDataframePartitionManager.wait_partitions" | ||
| ) | ||
| elif engine == "Unidist": | ||
| wait_method = ( | ||
| "modin.core.execution.unidist.implementations." | ||
| + "pandas_on_unidist.partitioning." | ||
| + "PandasOnUnidistDataframePartitionManager.wait_partitions" | ||
| ) | ||
| else: | ||
| wait_method = ( | ||
| "modin.core.dataframe.pandas.partitioning." | ||
| + "partition_manager.PandasDataframePartitionManager.wait_partitions" | ||
| import sys | ||
| import mpi4py | ||
| mpi4py.rc(recv_mprobe=False, initialize=False) | ||
| from mpi4py import MPI # noqa: E402 | ||
|
|
||
| ######################## | ||
| # Threads initializing # | ||
| ######################## | ||
|
|
||
| MPI.Init_thread() | ||
|
|
||
| comm = MPI.COMM_WORLD | ||
| rank = comm.Get_rank() | ||
| size = comm.Get_size() | ||
|
|
||
| parent_comm = MPI.Comm.Get_parent() | ||
|
|
||
| if rank == 0 and parent_comm == MPI.COMM_NULL and size == 1: | ||
| nprocs_to_spawn = 4 | ||
| args = ["reproducer.py"] | ||
| info = MPI.Info.Create() | ||
| print("main before spawn") | ||
| intercomm = MPI.COMM_SELF.Spawn( | ||
| sys.executable, | ||
| args, | ||
| maxprocs=nprocs_to_spawn, | ||
| info=info, | ||
| root=rank, | ||
| ) | ||
|
|
||
| ####################### | ||
| # Merge communicators # | ||
| ####################### | ||
|
|
||
| @pytest.mark.parametrize("set_benchmark_mode", [False], indirect=True) | ||
| def test_turn_off(set_benchmark_mode): | ||
| df = pd.DataFrame([0]) | ||
| with mock.patch(wait_method) as wait: | ||
| df.dropna() | ||
| wait.assert_not_called() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("set_benchmark_mode", [True], indirect=True) | ||
| def test_turn_on(set_benchmark_mode): | ||
| df = pd.DataFrame([0]) | ||
| with mock.patch(wait_method) as wait: | ||
| df.dropna() | ||
| wait.assert_called() | ||
| # Just comment the foollowing block if you want to run without communicators merging. | ||
| if parent_comm != MPI.COMM_NULL: | ||
| print("child before merge") | ||
| comm = parent_comm.Merge(high=True) | ||
| else: | ||
| print("main before merge") | ||
| comm = intercomm.Merge(high=False) | ||
|
|
||
| # rank = comm.Get_rank() | ||
| # size = comm.Get_size() | ||
|
|
||
| # if rank == 0: | ||
| # print(f"size = {size}") | ||
| # data = {"a": 1} | ||
| # for r in range(2, size): | ||
| # comm.send(data, dest=r, tag=11) | ||
| # comm.send(data, dest=r, tag=12) | ||
| # data2 = comm.recv(source=r, tag=11) | ||
| # data2 = comm.recv(source=r, tag=12) | ||
| # elif rank == 1: | ||
|
|
||
| # else: | ||
| # data = comm.recv(source=0, tag=11) | ||
| # data = comm.recv(source=0, tag=12) | ||
| # comm.send(data, dest=0, tag=11) | ||
| # comm.send(data, dest=0, tag=12) | ||
|
|
||
| if not MPI.Is_finalized(): | ||
| MPI.Finalize() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters