Move all commands to separate script files

.github/workflows/freebsd.yml

@@ -27,8 +27,5 @@ jobs:
             pkg install -y cmake git ninja googletest
 
           run: |
-            mkdir build
-            cd build
-            cmake .. -GNinja -DGOOGLE_TEST=ON
-            ninja -v
-            ./testxgboost
+            bash ops/script/build_via_cmake.sh
+            ./build/testxgboost

.github/workflows/jvm_tests.yml

@@ -232,7 +232,6 @@ jobs:
         run: |
           cd lib/
           Rename-Item -Path xgboost4j.dll -NewName xgboost4j_${{ github.sha }}.dll
-          dir
           python -m awscli s3 cp xgboost4j_${{ github.sha }}.dll `
             s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/ `
             --acl public-read --region us-west-2

.github/workflows/main.yml

@@ -268,14 +268,14 @@ jobs:
           KEY: ${{ matrix.artifact_from }}
       - run: bash ops/pipeline/test-cpp-gpu.sh ${{ matrix.suite }}
 
-  test-python:
+  test-python-wheel:
     name: Run Python tests (${{ matrix.description }})
     needs: [build-cuda, build-cpu-arm64]
     runs-on:
       - runs-on
       - runner=${{ matrix.runner }}
       - run-id=${{ github.run_id }}
-      - tag=main-test-python-${{ matrix.description }}
+      - tag=main-test-python-wheel-${{ matrix.description }}
     strategy:
       fail-fast: false
       max-parallel: 2
@@ -329,4 +329,4 @@ jobs:
           COMMAND: download
           KEY: ${{ matrix.artifact_from }}
       - name: Run Python tests, ${{ matrix.description }}
-        run: bash ops/pipeline/test-python.sh ${{ matrix.suite }} ${{ matrix.container }}
+        run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.container }}

.github/workflows/misc.yml

@@ -14,28 +14,6 @@ env:
     ${{ github.event.pull_request.number && 'PR-' }}${{ github.event.pull_request.number || github.ref_name }}
 
 jobs:
-  gtest-cpu:
-    name: Test Google C++ test (CPU)
-    runs-on: macos-13
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - name: Install system packages
-        run: |
-          brew install ninja libomp
-      - name: Build gtest binary
-        run: |
-          mkdir build
-          cd build
-          cmake .. -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON -GNinja -DBUILD_DEPRECATED_CLI=ON -DUSE_SANITIZER=ON -DENABLED_SANITIZERS=address -DCMAKE_BUILD_TYPE=RelWithDebInfo
-          ninja -v
-      - name: Run gtest binary
-        run: |
-          cd build
-          ./testxgboost
-          ctest -R TestXGBoostCLI --extra-verbose
-
   gtest-cpu-nonomp:
     name: Test Google C++ unittest (CPU Non-OMP)
     runs-on: ubuntu-latest
@@ -47,12 +25,7 @@ jobs:
         run: |
           sudo apt-get install -y --no-install-recommends ninja-build
       - name: Build and install XGBoost
-        shell: bash -l {0}
-        run: |
-          mkdir build
-          cd build
-          cmake .. -GNinja -DGOOGLE_TEST=ON  -DUSE_DMLC_GTEST=ON -DUSE_OPENMP=OFF -DBUILD_DEPRECATED_CLI=ON
-          ninja -v
+        run: bash ops/script/build_via_cmake.sh -DUSE_OPENMP=OFF
       - name: Run gtest binary
         run: |
           cd build
@@ -79,42 +52,5 @@ jobs:
         run: |
           conda info
           conda list
-      - name: Build and install XGBoost static library
-        run: |
-          mkdir build
-          cd build
-          cmake .. -DBUILD_STATIC_LIB=ON -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja
-          ninja -v install
-          cd -
-      - name: Build and run C API demo with static
-        run: |
-          pushd .
-          cd demo/c-api/
-          mkdir build
-          cd build
-          cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
-          ninja -v
-          ctest
-          cd ..
-          rm -rf ./build
-          popd
-
-      - name: Build and install XGBoost shared library
-        run: |
-          cd build
-          cmake .. -DBUILD_STATIC_LIB=OFF -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja -DPLUGIN_FEDERATED=ON -DGOOGLE_TEST=ON
-          ninja -v install
-          ./testxgboost
-          cd -
       - name: Build and run C API demo with shared
-        run: |
-          pushd .
-          cd demo/c-api/
-          mkdir build
-          cd build
-          cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
-          ninja -v
-          ctest
-          popd
-          ./ops/script/verify_link.sh ./demo/c-api/build/basic/api-demo
-          ./ops/script/verify_link.sh ./demo/c-api/build/external-memory/external-memory-demo
+        run: bash ops/pipeline/test-c-api-demo.sh

.github/workflows/python_tests.yml

@@ -14,9 +14,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  python-sdist-test-on-Linux:
-    runs-on: ubuntu-latest
-    name: Test installing XGBoost Python source package
+  python-sdist-test:
+    runs-on: ${{ matrix.os }}
+    name: Test installing Python XGBoost from the source distribution (${{ matrix.os }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-13, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -32,53 +36,12 @@ jobs:
         run: |
           conda info
           conda list
-      - name: Build and install XGBoost
+      - name: Install extra package for MacOS
         run: |
-          cd python-package
-          python --version
-          python -m build --sdist
-          pip install -v ./dist/xgboost-*.tar.gz --config-settings use_openmp=False
-          cd ..
-          python -c 'import xgboost'
-
-  python-sdist-test:
-    # Use system toolchain instead of conda toolchain for macos and windows.
-    # MacOS has linker error if clang++ from conda-forge is used
-    runs-on: ${{ matrix.os }}
-    name: Test installing XGBoost Python source package on ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-13, windows-latest]
-        python-version: ["3.10"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - name: Install osx system dependencies
+          mamba install -c conda-forge llvm-openmp
         if: matrix.os == 'macos-13'
-        run: |
-          brew install ninja libomp
-      - uses: conda-incubator/setup-miniconda@v3
-        with:
-          auto-update-conda: true
-          python-version: ${{ matrix.python-version }}
-          activate-environment: test
-      - name: Install build
-        run: |
-          conda install -c conda-forge python-build
-      - name: Display Conda env
-        run: |
-          conda info
-          conda list
       - name: Build and install XGBoost
-        run: |
-          cd python-package
-          python --version
-          python -m build --sdist
-          pip install -v ./dist/xgboost-*.tar.gz
-          cd ..
-          python -c 'import xgboost'
+        run: bash ops/pipeline/test-python-sdist.sh
 
   python-tests-on-macos:
     name: Test XGBoost Python package on macos-13
@@ -102,31 +65,7 @@ jobs:
           conda info
           conda list
 
-      - name: Build XGBoost on macos
-        run: |
-          brew install ninja
-
-          mkdir build
-          cd build
-          # Set prefix, to use OpenMP library from Conda env
-          # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
-          # to learn why we don't use libomp from Homebrew.
-          cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DBUILD_DEPRECATED_CLI=ON
-          ninja
-
-      - name: Install Python package
-        run: |
-          cd python-package
-          python --version
-          pip install -v .
-
-      - name: Test Python package
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/python
-
-      - name: Test Dask Interface
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask
+      - run: bash ops/pipeline/test-python-macos.sh
 
   python-system-installation-on-ubuntu:
     name: Test XGBoost Python package System Installation on Ubuntu
@@ -135,32 +74,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-
       - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-
-      - name: Install ninja
-        run: |
-          sudo apt-get update && sudo apt-get install -y ninja-build
-
-      - name: Build XGBoost on Ubuntu
-        run: |
-          mkdir build
-          cd build
-          cmake .. -GNinja
-          ninja
-
-      - name: Copy lib to system lib
-        run: |
-          cp lib/* "$(python -c 'import sys; print(sys.base_prefix)')/lib"
-
-      - name: Install XGBoost in Virtual Environment
-        run: |
-          cd python-package
-          pip install virtualenv
-          virtualenv venv
-          source venv/bin/activate && \
-            pip install -v . --config-settings use_system_libxgboost=True && \
-            python -c 'import xgboost'
+      - run: bash ops/pipeline/test-python-with-sysprefix.sh

.github/workflows/sycl_tests.yml

@@ -38,15 +38,9 @@ jobs:
         conda list
     - name: Build and install XGBoost
       run: |
-        mkdir build
-        cd build
-        cmake .. -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DPLUGIN_SYCL=ON -DCMAKE_CXX_COMPILER=g++ \
-          -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja
-        ninja
+        bash ops/script/build_via_cmake.sh --conda-env=linux_sycl_test -DPLUGIN_SYCL=ON
     - name: Run gtest
-      run: |
-        cd build
-        ./testxgboost
+      run: ./build/testxgboost
 
   python-sycl-tests-on-ubuntu:
     name: Test XGBoost Python package with SYCL
@@ -56,31 +50,22 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-
       - uses: conda-incubator/setup-miniconda@v3
         with:
           miniforge-variant: Miniforge3
           miniforge-version: latest
           activate-environment: linux_sycl_test
           environment-file: ops/conda_env/linux_sycl_test.yml
           use-mamba: true
-
       - name: Display Conda env
         run: |
           conda info
           conda list
-      - name: Build XGBoost on Ubuntu
-        run: |
-          mkdir build
-          cd build
-          cmake .. -DPLUGIN_SYCL=ON -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -GNinja
-          ninja
-      - name: Install Python package
+      - name: Build and install XGBoost
         run: |
+          bash ops/script/build_via_cmake.sh --conda-env=linux_sycl_test -DPLUGIN_SYCL=ON
           cd python-package
           python --version
           pip install -v .
       - name: Test Python package
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/python-sycl/
+        run: pytest -s -v -rxXs --durations=0 ./tests/python-sycl/

ops/conda_env/sdist_test.yml

@@ -9,5 +9,3 @@ dependencies:
 - cmake
 - ninja
 - python-build
-- c-compiler
-- cxx-compiler

ops/pipeline/build-test-jvm-packages-impl.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 ## Build and test JVM packages.
+## Companion script for build-test-jvm-packages.sh.
 ##
 ## Note. This script takes in all inputs via environment variables.
 

ops/pipeline/classify-git-branch.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-
-## Detect whether the current git branch is a pull request or a release branch.
+## Detect whether the current git branch is a pull request or a release branch
 
 set -euo pipefail
 

ops/pipeline/deploy-jvm-packages-impl.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-## Deploy JVM packages to xgboost-maven-repo S3 bucket
+## Deploy JVM packages to S3 bucket
+## Companion script for ops/pipeline/deploy-jvm-packages.sh
 
 set -euox pipefail
 

ops/pipeline/deploy-jvm-packages.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+## Deploy JVM packages to S3 bucket
 
 set -euox pipefail
 

ops/pipeline/test-c-api-demo.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+## Test C API demos
+
+set -euox pipefail
+
+# Build and install XGBoost static library (libxgboost.a)
+mkdir build
+pushd build
+cmake .. -DBUILD_STATIC_LIB=ON -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja
+ninja -v install
+popd
+
+# Build and run C API demo with static library
+pushd demo/c-api/
+mkdir build-c-api-demo
+pushd build-c-api-demo
+cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
+ninja -v
+ctest
+popd
+rm -rf ./build-c-api-demo
+popd
+
+# Build and install XGBoost shared library (libxgboost.so)
+pushd build
+cmake .. -DBUILD_STATIC_LIB=OFF -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja \
+  -DPLUGIN_FEDERATED=ON
+ninja -v install
+popd
+
+# Build and run C API demo with shared library
+mkdir demo/c-api/build-c-api-demo
+pushd demo/c-api/build-c-api-demo
+cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX
+ninja -v
+ctest
+popd
+./ops/script/verify_link.sh ./demo/c-api/build-c-api-demo/basic/api-demo
+./ops/script/verify_link.sh ./demo/c-api/build-c-api-demo/external-memory/external-memory-demo