switch nose with pytest

.github/workflows/os_x_staticbuild.yml

@@ -10,16 +10,16 @@ jobs:
         uses: actions/checkout@v2
       - name: Install Dependencies
         run: |
-          brew install nasm automake ninja libtool cmake pkgconfig protobuf
+          brew install nasm automake ninja libtool cmake pkgconfig protobuf hdf5 zlib
+          python3 -m pip install --user -r ci/docker/install/requirements
       - name: Build project
         run: |
           git --version
           clang --version
           CMAKE_STATICBUILD=1 ./tools/staticbuild/build.sh cpu
       - name: Setup Python
         run: |
-          python3 -m pip install --user nose nose-timer nose-exclude numpy scipy
           python3 -m pip install --user -e python
       - name: Test project
         run: |
-          python3 -m nose --with-timer --verbose tests/python/unittest/ --exclude-test=test_extensions.test_subgraph --exclude-test=test_extensions.test_custom_op --exclude-test=test_gluon_data.test_recordimage_dataset_with_data_loader_multiworker --exclude-test=test_gluon_data.test_multi_worker --exclude-test=test_gluon_data.test_multi_worker_shape --exclude-test=test_gluon_data.test_multi_worker_forked_data_loader --exclude-test=test_gluon_data.test_multi_worker_dataloader_release_pool
+          python3 -m pytest --durations=50 --verbose tests/python/unittest/ -k 'not (test_subgraph or test_custom_op or test_recordimage_dataset_with_data_loader_multiworker or test_multi_worker or test_multi_worker_shape or test_multi_worker_forked_data_loader or test_multi_worker_dataloader_release_pool)'

cd/mxnet_lib/mxnet_lib_pipeline.groovy

@@ -98,7 +98,7 @@ def unittest_py3(mxnet_variant) {
       def image = get_environment(mxnet_variant)
       def use_nvidia_docker = mxnet_variant.startsWith('cu')
       ci_utils.unpack_and_init("mxnet_${mxnet_variant}", get_stash(mxnet_variant), false)
-      ci_utils.docker_run(image, "cd_unittest_ubuntu ${mxnet_variant} python3", use_nvidia_docker)
+      ci_utils.docker_run(image, "cd_unittest_ubuntu ${mxnet_variant}", use_nvidia_docker)
     }
   }
 }

cd/python/docker/Dockerfile.test

@@ -24,7 +24,7 @@ ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 
 # Install test dependencies
-RUN pip install nose
+RUN pip install pytest
 
 ARG USER_ID=1001
 ARG GROUP_ID=1001

cd/python/pypi/Jenkins_pipeline.groovy

@@ -58,7 +58,7 @@ def test(mxnet_variant) {
     // test wheel file
     def environment = get_environment(mxnet_variant)
     def nvidia_docker = mxnet_variant.startsWith('cu')
-    ci_utils.docker_run(environment, "cd_integration_test_pypi python3 ${nvidia_docker}", nvidia_docker)
+    ci_utils.docker_run(environment, "cd_integration_test_pypi ${nvidia_docker}", nvidia_docker)
   }
 }
 

ci/dev_menu.py

@@ -105,8 +105,8 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
         # Install MXNet python bindigs
         check_call([pip, 'install', '--upgrade', '--force-reinstall', '-e', 'python'])
         # Install test dependencies
-        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r', os.path.join('tests',
-            'requirements.txt')])
+        check_call([pip, 'install', '--upgrade', '--force-reinstall', '-r',
+                    os.path.join('ci', 'docker', 'install', 'requirements')])
     else:
         logging.warn("Can't find pip: '%s' not found", pip)
 
@@ -119,7 +119,7 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
         provision_virtualenv,
     ]),
     ('[Local] Python Unit tests',
-        "./py3_venv/bin/nosetests -v tests/python/unittest/"
+        "pytest -v tests/python/unittest/"
     ),
     ('[Docker] Build the MXNet binary - outputs to "lib/"',
         "ci/build.py --platform ubuntu_cpu_lite /work/runtime_functions.sh build_ubuntu_cpu_docs"),

ci/docker/Dockerfile.build.centos7_cpu

@@ -27,13 +27,14 @@ RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/
+COPY install/requirements /work/
 RUN /work/centos7_python.sh
 COPY install/centos7_scala.sh /work/
 RUN /work/centos7_scala.sh
 
 ARG USER_ID=0
 COPY install/centos7_adduser.sh /work/
-RUN /work/centos7_adduser.sh 
+RUN /work/centos7_adduser.sh
 
 ENV PYTHONPATH=./python/
 WORKDIR /work/mxnet

ci/docker/Dockerfile.build.centos7_gpu

@@ -27,6 +27,7 @@ RUN /work/centos7_core.sh
 COPY install/centos7_ccache.sh /work/
 RUN /work/centos7_ccache.sh
 COPY install/centos7_python.sh /work/
+COPY install/requirements /work/
 RUN /work/centos7_python.sh
 
 ENV CUDNN_VERSION=7.6.0.64

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -54,12 +54,6 @@ RUN /work/ubuntu_gcc8.sh
 COPY install/ubuntu_mkl.sh /work/
 RUN /work/ubuntu_mkl.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_cpu_julia

@@ -54,12 +54,6 @@ RUN /work/ubuntu_gcc8.sh
 COPY install/ubuntu_mkl.sh /work/
 RUN /work/ubuntu_mkl.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_cpu_python

@@ -32,9 +32,6 @@ COPY install/ubuntu_python.sh /work/
 COPY install/requirements /work/
 RUN /work/ubuntu_python.sh
 
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_docs.sh /work/
 RUN /work/ubuntu_docs.sh
 
@@ -46,4 +43,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.ubuntu_cpu_scala

@@ -50,4 +50,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/mxnet
+WORKDIR /work/mxnet

ci/docker/Dockerfile.build.ubuntu_gpu_cu101

@@ -51,12 +51,6 @@ RUN /work/ubuntu_tvm.sh
 COPY install/ubuntu_llvm.sh /work/
 RUN /work/ubuntu_llvm.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_docs.sh /work/
 COPY install/requirements /work/
 RUN /work/ubuntu_docs.sh

ci/docker/Dockerfile.build.ubuntu_nightly_cpu

@@ -45,12 +45,6 @@ RUN /work/ubuntu_clang.sh
 COPY install/ubuntu_gcc8.sh /work/
 RUN /work/ubuntu_gcc8.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/Dockerfile.build.ubuntu_nightly_gpu

@@ -51,12 +51,6 @@ RUN /work/ubuntu_tvm.sh
 COPY install/ubuntu_llvm.sh /work/
 RUN /work/ubuntu_llvm.sh
 
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh

ci/docker/install/centos7_python.sh

@@ -29,4 +29,4 @@ yum -y install python36u
 # Install PIP
 curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
 python3.6 get-pip.py
-pip3 install nose pylint numpy nose-timer requests h5py scipy==1.2.3
+pip3 install -r /work/requirements

ci/docker/install/requirements

@@ -22,13 +22,19 @@ boto3==1.9.229
 cpplint==1.3.0
 Cython==0.29.7
 decorator==4.4.0
-h5py==2.8.0rc1
+h5py==2.10.0
 mock==2.0.0
-nose==1.3.7
-nose-timer==0.7.3
 numpy>1.16.0,<2.0.0
 pylint==2.3.1; python_version >= '3.0'
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
 six==1.11.0
 setuptools
+pytest==5.3.2
+pytest-env==0.6.2
+pytest-cov==2.8.1
+pytest-xdist==1.31.0
+protobuf==3.5.2
+onnx==1.5.0
+Pillow<6
+tabulate==0.7.5

ci/docker/install/ubuntu_python.sh

@@ -24,6 +24,7 @@ set -ex
 # install libraries for mxnet's python package on ubuntu
 apt-get update || true
 apt-get install -y python-dev python3-dev virtualenv wget
+apt-get install -y libprotobuf-dev protobuf-compiler
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py