Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CI] Fix JVM tests on GH Action #10064

Merged
merged 22 commits into from
Feb 22, 2024
Merged
32 changes: 18 additions & 14 deletions .github/workflows/jvm_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,31 +15,35 @@ jobs:
os: [windows-latest, ubuntu-latest, macos-11]

steps:
- uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
submodules: 'true'

- uses: actions/setup-python@7f80679172b057fc5e90d70d197929d454754a5a # v4.3.0
- uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7 # v1.8.1
with:
python-version: '3.8'
architecture: 'x64'

- uses: actions/setup-java@d202f5dbf7256730fb690ec59f6381650114feb2 # v3.6.0
with:
java-version: 1.8

- name: Install Python packages
run: |
python -m pip install wheel setuptools
python -m pip install awscli
micromamba-version: '1.5.6-0'
environment-name: jvm_tests
extra-specs: >-
python=3.10
awscli
cache-downloads: true
cache-env: true

- name: Cache Maven packages
uses: actions/cache@6998d139ddd3e68c71e9e398d8e40b71a2f39812 # v3.2.5
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}
restore-keys: ${{ runner.os }}-m2-${{ hashFiles('./jvm-packages/pom.xml') }}

- name: Build xgboost4j.dll
run: |
mkdir build
cd build
cmake .. -G"Visual Studio 17 2022" -A x64 -DJVM_BINDINGS=ON
cmake --build . --config Release
if: matrix.os == 'windows-latest'

- name: Test XGBoost4J (Core)
run: |
cd jvm-packages
Expand Down
65 changes: 40 additions & 25 deletions jvm-packages/create_jni.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,40 +81,55 @@ def native_build(args):
with cd(".."):
build_dir = "build-gpu" if cli_args.use_cuda == "ON" else "build"
maybe_makedirs(build_dir)
with cd(build_dir):
if sys.platform == "win32":
# Force x64 build on Windows.
maybe_generator = " -A x64"
else:
maybe_generator = ""
if sys.platform == "linux":
maybe_parallel_build = " -- -j $(nproc)"
else:
maybe_parallel_build = ""

if cli_args.log_capi_invocation == "ON":
CONFIG["LOG_CAPI_INVOCATION"] = "ON"
if sys.platform == "linux":
maybe_parallel_build = " -- -j $(nproc)"
else:
maybe_parallel_build = ""

if cli_args.log_capi_invocation == "ON":
CONFIG["LOG_CAPI_INVOCATION"] = "ON"

if cli_args.use_cuda == "ON":
CONFIG["USE_CUDA"] = "ON"
CONFIG["USE_NCCL"] = "ON"
CONFIG["USE_DLOPEN_NCCL"] = "OFF"
if cli_args.use_cuda == "ON":
CONFIG["USE_CUDA"] = "ON"
CONFIG["USE_NCCL"] = "ON"
CONFIG["USE_DLOPEN_NCCL"] = "OFF"

args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()]
args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()]

# if enviorment set rabit_mock
if os.getenv("RABIT_MOCK", None) is not None:
args.append("-DRABIT_MOCK:BOOL=ON")
# if enviorment set rabit_mock
if os.getenv("RABIT_MOCK", None) is not None:
args.append("-DRABIT_MOCK:BOOL=ON")

# if enviorment set GPU_ARCH_FLAG
gpu_arch_flag = os.getenv("GPU_ARCH_FLAG", None)
if gpu_arch_flag is not None:
args.append("%s" % gpu_arch_flag)
# if enviorment set GPU_ARCH_FLAG
gpu_arch_flag = os.getenv("GPU_ARCH_FLAG", None)
if gpu_arch_flag is not None:
args.append("%s" % gpu_arch_flag)

with cd(build_dir):
lib_dir = os.path.join(os.pardir, "lib")
if os.path.exists(lib_dir):
shutil.rmtree(lib_dir)
run("cmake .. " + " ".join(args) + maybe_generator)

# Same trick as Python build, just test all possible generators.
if sys.platform == "win32":
supported_generators = (
"", # empty, decided by cmake
'-G"Visual Studio 17 2022" -A x64',
'-G"Visual Studio 16 2019" -A x64',
'-G"Visual Studio 15 2017" -A x64',
)
for generator in supported_generators:
try:
run("cmake .. " + " ".join(args + [generator]))
break
except subprocess.CalledProcessError as e:
print(f"Failed to build with generator: {generator}", e)
with cd(os.path.pardir):
shutil.rmtree(build_dir)
maybe_makedirs(build_dir)
else:
run("cmake .. " + " ".join(args))
run("cmake --build . --config Release" + maybe_parallel_build)

with cd("demo/CLI/regression"):
Expand Down
9 changes: 6 additions & 3 deletions python-package/packager/nativelib.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@ def build_libxgboost(
build_dir: pathlib.Path,
build_config: BuildConfiguration,
) -> pathlib.Path:
"""Build libxgboost in a temporary directory and obtain the path to built libxgboost"""
"""Build libxgboost in a temporary directory and obtain the path to built
libxgboost.

"""
logger = logging.getLogger("xgboost.packager.build_libxgboost")

if not cpp_src_dir.is_dir():
Expand All @@ -51,8 +54,8 @@ def _build(*, generator: str) -> None:
cmake_cmd.extend(build_config.get_cmake_args())

# Flag for cross-compiling for Apple Silicon
# We use environment variable because it's the only way to pass down custom flags
# through the cibuildwheel package, which calls `pip wheel` command.
# We use environment variable because it's the only way to pass down custom
# flags through the cibuildwheel package, which calls `pip wheel` command.
if "CIBW_TARGET_OSX_ARM64" in os.environ:
cmake_cmd.append("-DCMAKE_OSX_ARCHITECTURES=arm64")

Expand Down
Loading