From 34d9921dd52140bb7c09c9edd5b997b1b3d6d5b2 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Fri, 20 Mar 2020 04:04:43 -0700 Subject: [PATCH 1/3] De-duplicate macro _CRT_SECURE_NO_WARNINGS / _CRT_SECURE_NO_DEPRECATE --- include/rabit/internal/utils.h | 4 ++++ src/allreduce_base.cc | 4 ++++ src/allreduce_robust.cc | 4 ++++ src/c_api.cc | 4 ++++ src/engine.cc | 4 ++++ src/engine_base.cc | 4 ++++ src/engine_empty.cc | 4 ++++ src/engine_mock.cc | 4 ++++ src/engine_mpi.cc | 4 ++++ 9 files changed, 36 insertions(+) diff --git a/include/rabit/internal/utils.h b/include/rabit/internal/utils.h index 05f35557..a9c3a8f7 100644 --- a/include/rabit/internal/utils.h +++ b/include/rabit/internal/utils.h @@ -6,7 +6,11 @@ */ #ifndef RABIT_INTERNAL_UTILS_H_ #define RABIT_INTERNAL_UTILS_H_ + +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS + #include #include #include diff --git a/src/allreduce_base.cc b/src/allreduce_base.cc index 20f9318b..f0ae834c 100644 --- a/src/allreduce_base.cc +++ b/src/allreduce_base.cc @@ -5,8 +5,12 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include #include diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc index 8ae1acc5..85d7d5b0 100644 --- a/src/allreduce_robust.cc +++ b/src/allreduce_robust.cc @@ -5,8 +5,12 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include #include diff --git a/src/c_api.cc b/src/c_api.cc index d8533198..fa04a1f4 100644 --- a/src/c_api.cc +++ b/src/c_api.cc @@ -1,7 +1,11 @@ // Copyright by Contributors // implementations in ctypes +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #include #include diff --git a/src/engine.cc b/src/engine.cc index 4701d2fd..67b22b59 100644 --- a/src/engine.cc +++ b/src/engine.cc @@ -6,8 +6,12 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include diff --git a/src/engine_base.cc b/src/engine_base.cc index 39da566d..57a88968 100644 --- a/src/engine_base.cc +++ b/src/engine_base.cc @@ -6,8 +6,12 @@ * \author Tianqi Chen */ // define use MOCK, os we will use mock Manager +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX // switch engine to AllreduceMock #define RABIT_USE_BASE diff --git a/src/engine_empty.cc b/src/engine_empty.cc index 5cecc03c..2797f54f 100644 --- a/src/engine_empty.cc +++ b/src/engine_empty.cc @@ -6,8 +6,12 @@ * This is usually NOT needed, use engine_mpi or engine for real distributed version * \author Tianqi Chen */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include "rabit/internal/engine.h" diff --git a/src/engine_mock.cc b/src/engine_mock.cc index aab012db..43119e61 100644 --- a/src/engine_mock.cc +++ b/src/engine_mock.cc @@ -6,8 +6,12 @@ * \author Tianqi Chen */ // define use MOCK, os we will use mock Manager +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX // switch engine to AllreduceMock #define RABIT_USE_MOCK diff --git a/src/engine_mpi.cc b/src/engine_mpi.cc index f6177080..b31d6014 100644 --- a/src/engine_mpi.cc +++ b/src/engine_mpi.cc @@ -6,8 +6,12 @@ * * \author Tianqi Chen */ +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE #define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX #include #include From ef3fb2d6f45f143c61972b388fc284f4ac9d546b Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Sat, 27 Jun 2020 04:43:46 -0700 Subject: [PATCH 2/3] Move all macros to base.h --- include/rabit/base.h | 19 +++++++++++++++++++ include/rabit/internal/utils.h | 5 +---- src/allreduce_base.cc | 7 +------ src/allreduce_robust.cc | 7 +------ src/c_api.cc | 8 +------- src/engine.cc | 9 +-------- src/engine_base.cc | 7 +------ src/engine_empty.cc | 7 +------ src/engine_mock.cc | 7 +------ src/engine_mpi.cc | 9 ++------- 10 files changed, 29 insertions(+), 56 deletions(-) create mode 100644 include/rabit/base.h diff --git a/include/rabit/base.h b/include/rabit/base.h new file mode 100644 index 00000000..ab3a285d --- /dev/null +++ b/include/rabit/base.h @@ -0,0 +1,19 @@ +/*! + * Copyright (c) 2020 by Contributors + * \file base.h + * \brief Macros common to all headers + * + * \author Hyunsu Cho + */ + +#ifndef RABIT_BASE_H_ +#define RABIT_BASE_H_ + +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif // _CRT_SECURE_NO_WARNINGS +#ifndef _CRT_SECURE_NO_DEPRECATE +#define _CRT_SECURE_NO_DEPRECATE +#endif // _CRT_SECURE_NO_DEPRECATE + +#endif // RABIT_BASE_H_ diff --git a/include/rabit/internal/utils.h b/include/rabit/internal/utils.h index a9c3a8f7..918a913c 100644 --- a/include/rabit/internal/utils.h +++ b/include/rabit/internal/utils.h @@ -7,10 +7,7 @@ #ifndef RABIT_INTERNAL_UTILS_H_ #define RABIT_INTERNAL_UTILS_H_ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS - +#include #include #include #include diff --git a/src/allreduce_base.cc b/src/allreduce_base.cc index f0ae834c..179a11f7 100644 --- a/src/allreduce_base.cc +++ b/src/allreduce_base.cc @@ -5,13 +5,8 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX +#include #include #include #include diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc index 85d7d5b0..1ce407d6 100644 --- a/src/allreduce_robust.cc +++ b/src/allreduce_robust.cc @@ -5,13 +5,8 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX +#include #include #include #include diff --git a/src/c_api.cc b/src/c_api.cc index 99523d46..b1c8689f 100644 --- a/src/c_api.cc +++ b/src/c_api.cc @@ -1,12 +1,6 @@ // Copyright by Contributors // implementations in ctypes -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE - +#include #include #include #include "rabit/rabit.h" diff --git a/src/engine.cc b/src/engine.cc index 67b22b59..69d68eb3 100644 --- a/src/engine.cc +++ b/src/engine.cc @@ -6,14 +6,7 @@ * * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou */ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE -#define NOMINMAX - +#include #include #include "rabit/internal/engine.h" #include "allreduce_base.h" diff --git a/src/engine_base.cc b/src/engine_base.cc index 57a88968..f73bbb2c 100644 --- a/src/engine_base.cc +++ b/src/engine_base.cc @@ -6,13 +6,8 @@ * \author Tianqi Chen */ // define use MOCK, os we will use mock Manager -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX +#include // switch engine to AllreduceMock #define RABIT_USE_BASE #include "engine.cc" diff --git a/src/engine_empty.cc b/src/engine_empty.cc index 2797f54f..b8e7e5ad 100644 --- a/src/engine_empty.cc +++ b/src/engine_empty.cc @@ -6,14 +6,9 @@ * This is usually NOT needed, use engine_mpi or engine for real distributed version * \author Tianqi Chen */ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX +#include #include "rabit/internal/engine.h" namespace rabit { diff --git a/src/engine_mock.cc b/src/engine_mock.cc index 43119e61..f38c423d 100644 --- a/src/engine_mock.cc +++ b/src/engine_mock.cc @@ -6,15 +6,10 @@ * \author Tianqi Chen */ // define use MOCK, os we will use mock Manager -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX // switch engine to AllreduceMock #define RABIT_USE_MOCK +#include #include "allreduce_mock.h" #include "engine.cc" diff --git a/src/engine_mpi.cc b/src/engine_mpi.cc index 93c3479c..23f7bc53 100644 --- a/src/engine_mpi.cc +++ b/src/engine_mpi.cc @@ -6,15 +6,10 @@ * * \author Tianqi Chen */ -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif // _CRT_SECURE_NO_WARNINGS -#ifndef _CRT_SECURE_NO_DEPRECATE -#define _CRT_SECURE_NO_DEPRECATE -#endif // _CRT_SECURE_NO_DEPRECATE #define NOMINMAX -#include +#include #include +#include #include "rabit/internal/engine.h" #include "rabit/internal/utils.h" From 32db01bcebf29a186d15acffe1cfa9d2a9b34e60 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Sat, 27 Jun 2020 05:04:10 -0700 Subject: [PATCH 3/3] Fix CI --- scripts/travis_osx_install.sh | 6 ------ scripts/travis_runtest.sh | 4 ++++ scripts/travis_setup.sh | 23 +++++++++++++++++++---- test/test.mk | 16 ++++++++-------- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/scripts/travis_osx_install.sh b/scripts/travis_osx_install.sh index 6775ba25..cae633d5 100755 --- a/scripts/travis_osx_install.sh +++ b/scripts/travis_osx_install.sh @@ -5,9 +5,3 @@ set -x if [ ${TRAVIS_OS_NAME} != "osx" ]; then exit 0 fi - -# Prevent clash between Python 2 and 3 -brew unlink python@2 -brew link --overwrite python - -python3 -m pip install --upgrade pip diff --git a/scripts/travis_runtest.sh b/scripts/travis_runtest.sh index 1ec04bb0..f3d63859 100755 --- a/scripts/travis_runtest.sh +++ b/scripts/travis_runtest.sh @@ -1,5 +1,9 @@ #!/bin/bash +conda activate python3 +conda --version +python --version + make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k || exit -1 make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_same || exit -1 make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_hard || exit -1 diff --git a/scripts/travis_setup.sh b/scripts/travis_setup.sh index d0b82bec..f03f400c 100755 --- a/scripts/travis_setup.sh +++ b/scripts/travis_setup.sh @@ -2,9 +2,24 @@ echo "Testing on: ${TRAVIS_OS_NAME}, Home directory: ${HOME}" -pip3 install cpplint pylint urllib3 numpy cpplint -pip3 install websocket-client kubernetes - +# Install Miniconda +if [ ${TRAVIS_OS_NAME} == "osx" ]; then + wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh +else + wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +fi +bash conda.sh -b -p $HOME/miniconda +source $HOME/miniconda/bin/activate +conda config --set always_yes yes --set changeps1 no +conda update -q conda +conda info -a +conda create -n python3 python=3.7 +conda activate python3 +conda --version +python --version +# Install Python packages +conda install -c conda-forge numpy scipy urllib3 websocket-client +python -m pip install cpplint pylint kubernetes # Install googletest under home directory GTEST_VERSION=1.8.1 @@ -27,7 +42,7 @@ make install popd if [ ${TRAVIS_OS_NAME} == "linux" ]; then - sudo apt-get install python3-pip tree + sudo apt-get install tree fi if [ ${TRAVIS_OS_NAME} == "osx" ]; then diff --git a/test/test.mk b/test/test.mk index 1028ec50..537212d1 100644 --- a/test/test.mk +++ b/test/test.mk @@ -13,25 +13,25 @@ all: model_recover_10_10k model_recover_10_10k_die_same model_recover_10_10k_di # this experiment test recovery with actually process exit, use keepalive to keep program alive model_recover_10_10k: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 rabit_bootstrap_cache=true rabit_debug=true rabit_reduce_ring_mincount=1 rabit_timeout=true rabit_timeout_sec=5 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 rabit_bootstrap_cache=true rabit_debug=true rabit_reduce_ring_mincount=1 rabit_timeout=true rabit_timeout_sec=5 model_recover_10_10k_die_same: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 rabit_bootstrap_cache=1 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 rabit_bootstrap_cache=1 model_recover_10_10k_die_hard: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 rabit_bootstrap_cache=1 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 rabit_bootstrap_cache=1 local_recover_10_10k: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1 pylocal_recover_10_10k: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover.py 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover.py 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1 lazy_recover_10_10k_die_hard: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 lazy_recover_10_10k_die_same: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 ringallreduce_10_10k: - $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 model_recover 10000 rabit_reduce_ring_mincount=10 + python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 model_recover 10000 rabit_reduce_ring_mincount=10