diff --git a/include/rabit/base.h b/include/rabit/base.h
new file mode 100644
index 00000000..ab3a285d
--- /dev/null
+++ b/include/rabit/base.h
@@ -0,0 +1,19 @@
+/*!
+ * Copyright (c) 2020 by Contributors
+ * \file base.h
+ * \brief Macros common to all headers
+ *
+ * \author Hyunsu Cho
+ */
+
+#ifndef RABIT_BASE_H_
+#define RABIT_BASE_H_
+
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif  // _CRT_SECURE_NO_WARNINGS
+#ifndef _CRT_SECURE_NO_DEPRECATE
+#define _CRT_SECURE_NO_DEPRECATE
+#endif  // _CRT_SECURE_NO_DEPRECATE
+
+#endif  // RABIT_BASE_H_
diff --git a/include/rabit/internal/utils.h b/include/rabit/internal/utils.h
index 05f35557..918a913c 100644
--- a/include/rabit/internal/utils.h
+++ b/include/rabit/internal/utils.h
@@ -6,7 +6,8 @@
  */
 #ifndef RABIT_INTERNAL_UTILS_H_
 #define RABIT_INTERNAL_UTILS_H_
-#define _CRT_SECURE_NO_WARNINGS
+
+#include <rabit/base.h>
 #include <string.h>
 #include <cstdio>
 #include <string>
diff --git a/scripts/travis_osx_install.sh b/scripts/travis_osx_install.sh
index 6775ba25..cae633d5 100755
--- a/scripts/travis_osx_install.sh
+++ b/scripts/travis_osx_install.sh
@@ -5,9 +5,3 @@ set -x
 if [ ${TRAVIS_OS_NAME} != "osx" ]; then
     exit 0
 fi
-
-# Prevent clash between Python 2 and 3
-brew unlink python@2
-brew link --overwrite python
-
-python3 -m pip install --upgrade pip
diff --git a/scripts/travis_runtest.sh b/scripts/travis_runtest.sh
index 1ec04bb0..f3d63859 100755
--- a/scripts/travis_runtest.sh
+++ b/scripts/travis_runtest.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+conda activate python3
+conda --version
+python --version
+
 make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k || exit -1
 make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_same  || exit -1
 make -f test.mk RABIT_BUILD_DMLC=1 model_recover_10_10k_die_hard || exit -1
diff --git a/scripts/travis_setup.sh b/scripts/travis_setup.sh
index d0b82bec..f03f400c 100755
--- a/scripts/travis_setup.sh
+++ b/scripts/travis_setup.sh
@@ -2,9 +2,24 @@
 
 echo "Testing on: ${TRAVIS_OS_NAME}, Home directory: ${HOME}"
 
-pip3 install cpplint pylint urllib3 numpy cpplint
-pip3 install websocket-client kubernetes
-
+# Install Miniconda
+if [ ${TRAVIS_OS_NAME} == "osx" ]; then
+  wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+else
+  wget -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+fi
+bash conda.sh -b -p $HOME/miniconda
+source $HOME/miniconda/bin/activate
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+conda info -a
+conda create -n python3 python=3.7
+conda activate python3
+conda --version
+python --version
+# Install Python packages
+conda install -c conda-forge numpy scipy urllib3 websocket-client
+python -m pip install cpplint pylint kubernetes
 
 # Install googletest under home directory
 GTEST_VERSION=1.8.1
@@ -27,7 +42,7 @@ make install
 popd
 
 if [ ${TRAVIS_OS_NAME} == "linux" ]; then
-    sudo apt-get install python3-pip tree
+    sudo apt-get install tree
 fi
 
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
diff --git a/src/allreduce_base.cc b/src/allreduce_base.cc
index 20f9318b..179a11f7 100644
--- a/src/allreduce_base.cc
+++ b/src/allreduce_base.cc
@@ -5,9 +5,8 @@
  *
  * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
  */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
+#include <rabit/base.h>
 #include <netinet/tcp.h>
 #include <cstring>
 #include <map>
diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc
index 8ae1acc5..1ce407d6 100644
--- a/src/allreduce_robust.cc
+++ b/src/allreduce_robust.cc
@@ -5,9 +5,8 @@
  *
  * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
  */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
+#include <rabit/base.h>
 #include <chrono>
 #include <thread>
 #include <limits>
diff --git a/src/c_api.cc b/src/c_api.cc
index 7459b7e0..b1c8689f 100644
--- a/src/c_api.cc
+++ b/src/c_api.cc
@@ -1,8 +1,6 @@
 // Copyright by Contributors
 // implementations in ctypes
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
-
+#include <rabit/base.h>
 #include <cstring>
 #include <string>
 #include "rabit/rabit.h"
diff --git a/src/engine.cc b/src/engine.cc
index 4701d2fd..69d68eb3 100644
--- a/src/engine.cc
+++ b/src/engine.cc
@@ -6,10 +6,7 @@
  *
  * \author Tianqi Chen, Ignacio Cano, Tianyi Zhou
  */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
-#define NOMINMAX
-
+#include <rabit/base.h>
 #include <memory>
 #include "rabit/internal/engine.h"
 #include "allreduce_base.h"
diff --git a/src/engine_base.cc b/src/engine_base.cc
index 39da566d..f73bbb2c 100644
--- a/src/engine_base.cc
+++ b/src/engine_base.cc
@@ -6,9 +6,8 @@
  * \author Tianqi Chen
  */
 // define use MOCK, os we will use mock Manager
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
+#include <rabit/base.h>
 // switch engine to AllreduceMock
 #define RABIT_USE_BASE
 #include "engine.cc"
diff --git a/src/engine_empty.cc b/src/engine_empty.cc
index 5cecc03c..b8e7e5ad 100644
--- a/src/engine_empty.cc
+++ b/src/engine_empty.cc
@@ -6,10 +6,9 @@
  *  This is usually NOT needed, use engine_mpi or engine for real distributed version
  * \author Tianqi Chen
  */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
 
+#include <rabit/base.h>
 #include "rabit/internal/engine.h"
 
 namespace rabit {
diff --git a/src/engine_mock.cc b/src/engine_mock.cc
index aab012db..f38c423d 100644
--- a/src/engine_mock.cc
+++ b/src/engine_mock.cc
@@ -6,11 +6,10 @@
  * \author Tianqi Chen
  */
 // define use MOCK, os we will use mock Manager
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
 // switch engine to AllreduceMock
 #define RABIT_USE_MOCK
+#include <rabit/base.h>
 #include "allreduce_mock.h"
 #include "engine.cc"
 
diff --git a/src/engine_mpi.cc b/src/engine_mpi.cc
index 7ff55b1e..23f7bc53 100644
--- a/src/engine_mpi.cc
+++ b/src/engine_mpi.cc
@@ -6,11 +6,10 @@
  *
  * \author Tianqi Chen
  */
-#define _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_DEPRECATE
 #define NOMINMAX
-#include <cstdio>
+#include <rabit/base.h>
 #include <mpi.h>
+#include <cstdio>
 #include "rabit/internal/engine.h"
 #include "rabit/internal/utils.h"
 
diff --git a/test/test.mk b/test/test.mk
index 1028ec50..537212d1 100644
--- a/test/test.mk
+++ b/test/test.mk
@@ -13,25 +13,25 @@ all: model_recover_10_10k  model_recover_10_10k_die_same model_recover_10_10k_di
 
 # this experiment test recovery with actually process exit, use keepalive to keep program alive
 model_recover_10_10k:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 rabit_bootstrap_cache=true rabit_debug=true rabit_reduce_ring_mincount=1 rabit_timeout=true rabit_timeout_sec=5
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 rabit_bootstrap_cache=true rabit_debug=true rabit_reduce_ring_mincount=1 rabit_timeout=true rabit_timeout_sec=5
 
 model_recover_10_10k_die_same:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 rabit_bootstrap_cache=1
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 rabit_bootstrap_cache=1
 
 model_recover_10_10k_die_hard:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 rabit_bootstrap_cache=1
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 model_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0 rabit_bootstrap_cache=1
 
 local_recover_10_10k:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
 
 pylocal_recover_10_10k:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover.py 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 local_recover.py 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=1,1,1,1
 
 lazy_recover_10_10k_die_hard:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=1,1,1,1 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0 mock=8,1,2,0 mock=4,1,3,0
 
 lazy_recover_10_10k_die_same:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 --local-num-attempt=20 lazy_recover 10000 mock=0,0,1,0 mock=1,1,1,0 mock=0,1,1,0 mock=4,1,1,0 mock=9,1,1,0
 
 ringallreduce_10_10k:
-	$(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 model_recover 10000 rabit_reduce_ring_mincount=10
+	python $(DMLC)/tracker/dmlc-submit --cluster local --num-workers=10 model_recover 10000 rabit_reduce_ring_mincount=10