Skip to content

Commit

Permalink
changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@arunjose696 @YarShev
arunjose696 authored and YarShev committed Oct 13, 2023
1 parent b608ea0 commit e0eae4a
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 16 deletions.
28 changes: 14 additions & 14 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:
run: |
MODIN_ENGINE=dask python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
MODIN_ENGINE=ray python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
MODIN_ENGINE=unidist UNIDIST_BACKEND=mpi mpiexec -n 1 python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
MODIN_ENGINE=unidist UNIDIST_BACKEND=mpi mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -c "import modin.pandas as pd; print(pd.DataFrame([1,2,3]))"
test-internals:
needs: [lint-flake8, lint-black-isort]
Expand Down Expand Up @@ -360,26 +360,26 @@ jobs:
run: |
sudo docker pull postgres
sudo docker run --name some-postgres -e POSTGRES_USER=sa -e POSTGRES_PASSWORD=Strong.Pwd-123 -e POSTGRES_DB=postgres -d -p 2345:5432 postgres
- run: mpiexec -n 1 python -m pytest modin/pandas/test/internals/test_benchmark_mode.py
- run: mpiexec -n 1 python -m pytest modin/pandas/test/internals/test_repartition.py
- run: mpiexec -n 1 python -m pytest modin/test/test_partition_api.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/pandas/test/internals/test_benchmark_mode.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/pandas/test/internals/test_repartition.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/test/test_partition_api.py
- uses: ./.github/actions/run-core-tests
with:
runner: mpiexec -n 1 python -m pytest
runner: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest
parallel: ""
- run: mpiexec -n 1 python -m pytest modin/numpy/test
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/numpy/test
- run: chmod +x ./.github/workflows/sql_server/set_up_sql_server.sh
- run: ./.github/workflows/sql_server/set_up_sql_server.sh
# need an extra argument "genv" to set environment variables for mpiexec. We need
# need an extra argument "genv" to set environment variables for mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist. We need
# these variables to test writing to the mock s3 filesystem.
- run: mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
- run: mpiexec -n 1 python -m pytest modin/experimental/pandas/test/test_io_exp.py
- run: mpiexec -n 1 python -m pytest modin/experimental/sql/test/test_sql.py
- run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/test_general.py
- run: mpiexec -n 1 python -m pytest modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe -x AWS_ACCESS_KEY_ID foobar_key -x AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest modin/pandas/test/test_io.py --verbose
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/experimental/pandas/test/test_io_exp.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/experimental/sql/test/test_sql.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/test/interchange/dataframe_protocol/test_general.py
- run: mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/test/interchange/dataframe_protocol/pandas/test_protocol.py
- run: |
python -m pip install lazy_import
mpiexec -n 1 python -m pytest modin/pandas/test/integrations/
mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe python -m pytest modin/pandas/test/integrations/
- uses: ./.github/actions/upload-coverage

test-all:
Expand Down Expand Up @@ -533,7 +533,7 @@ jobs:
shell-ex: "python -m pytest"
if: needs.execution-filter.dask != 'true'
- name: unidist
shell-ex: "mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest"
shell-ex: "mpiexec --prefix /usr/share/miniconda3/envs/modin_on_unidist -x UNIDIST_MPI_SHARED_OBJECT_STORE=True -n 1 --oversubscribe -x AWS_ACCESS_KEY_ID foobar_key -x AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest"
if: needs.execution-filter.unidist != 'true'
runs-on: ${{ matrix.os }}-latest
defaults:
Expand Down
7 changes: 5 additions & 2 deletions requirements/env_unidist_linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ dependencies:
# required dependencies
- pandas>=2.1,<2.2
- numpy>=1.22.4
- unidist-mpi>=0.2.1
- mpich
- cython
- openmpi
- mpi4py
- msgpack-python
- fsspec>=2022.05.0
- packaging>=21.0
- psutil>=5.8.0
Expand Down Expand Up @@ -59,3 +61,4 @@ dependencies:
- connectorx>=0.2.6a4
# The `numpydoc` version should match the version installed in the `lint-pydocstyle` job of the CI.
- numpydoc==1.1.0
- git+https://github.com/YarShev/unidist.git@dev/yigoshev-thr

0 comments on commit e0eae4a

Please sign in to comment.