diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml
index 314bc1286b23..bfa5bc297546 100644
--- a/.github/workflows/jvm_tests.yml
+++ b/.github/workflows/jvm_tests.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, ubuntu-latest]
+        os: [windows-latest, ubuntu-latest, macos-10.15]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
index f61dc132db66..9252de20c695 100644
--- a/.github/workflows/python_tests.yml
+++ b/.github/workflows/python_tests.yml
@@ -51,7 +51,8 @@ jobs:
     strategy:
       matrix:
         config:
-          - {os: windows-2016, compiler: 'msvc', python-version: '3.8'}
+          - {os: windows-2016, python-version: '3.8'}
+          - {os: macos-10.15, python-version "3.8" }
 
     steps:
     - uses: actions/checkout@v2
@@ -71,15 +72,27 @@ jobs:
         conda info
         conda list
 
-    - name: Build XGBoost with msvc
+    - name: Build XGBoost on Windows
       shell: bash -l {0}
-      if: matrix.config.compiler == 'msvc'
+      if: matrix.config.os == 'windows-2016'
       run: |
         mkdir build_msvc
         cd build_msvc
         cmake .. -G"Visual Studio 15 2017" -DCMAKE_CONFIGURATION_TYPES="Release" -A x64 -DGOOGLE_TEST=ON  -DUSE_DMLC_GTEST=ON
         cmake --build . --config Release --parallel $(nproc)
 
+    - name: Build XGBoost on macos
+      if: matrix.config.os == 'macos-10.15'
+      run: |
+        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
+        brew install ninja libomp
+        brew pin libomp
+
+        mkdir build
+        cd build
+        cmake .. -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
+        ninja
+
     - name: Install Python package
       shell: bash -l {0}
       run: |
@@ -92,3 +105,21 @@ jobs:
       shell: bash -l {0}
       run: |
         pytest -s -v ./tests/python
+
+    - name: Rename Python wheel
+      shell: bash -l {0}
+      if: matrix.config.os == 'macos-10.15'
+      run: |
+        TAG=macosx_10_15_x86_64.macosx_11_0_x86_64.macosx_12_0_x86_64
+        python tests/ci_build/rename_whl.py python-package/dist/*.whl ${{ github.sha }} ${TAG}
+
+    - name: Upload Python wheel
+      shell: bash -l {0}
+      if: |
+        (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_')) &&
+        matrix.os == 'macos-latest'
+      run: |
+        python -m awscli s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${{ steps.extract_branch.outputs.branch }}/ --acl public-read
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }}
diff --git a/.travis.yml b/.travis.yml
index a2d5966283bc..e8d4c3b9a9c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,14 +10,6 @@ env:
 
 jobs:
   include:
-    - os: osx
-      arch: amd64
-      osx_image: xcode10.2
-      env: TASK=python_test
-    - os: osx
-      arch: amd64
-      osx_image: xcode10.2
-      env: TASK=java_test
     - os: linux
       arch: s390x
       env: TASK=s390x_test
@@ -33,8 +25,6 @@ addons:
 
 before_install:
   - source tests/travis/travis_setup_env.sh
-  - if [ "${TASK}" != "python_sdist_test" ]; then export PYTHONPATH=${PYTHONPATH}:${PWD}/python-package; fi
-  - echo "MAVEN_OPTS='-Xmx2g -XX:MaxPermSize=1024m -XX:ReservedCodeCacheSize=512m -Dorg.slf4j.simpleLogger.defaultLogLevel=error'" > ~/.mavenrc
 
 install:
   - source tests/travis/setup.sh
diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh
index d3771f646dd3..405266e171a0 100755
--- a/tests/travis/setup.sh
+++ b/tests/travis/setup.sh
@@ -1,16 +1,5 @@
 #!/bin/bash
 
-if [ ${TRAVIS_OS_NAME} == "osx" ]; then
-    # https://travis-ci.community/t/macos-build-fails-because-of-homebrew-bundle-unknown-command/7296/27
-    # Use libomp 11.1.0: https://github.com/dmlc/xgboost/issues/7039
-    brew update  # Force update, so that update doesn't overwrite our version of libomp.rb
-    wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
-    brew install cmake libomp
-    brew pin libomp
-fi
-
-
-
 if [ ${TASK} == "python_test" ] || [ ${TASK} == "python_sdist_test" ]; then
     if [ ${TRAVIS_OS_NAME} == "osx" ]; then
         wget --no-verbose -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh