Use rapids-build-backend

.github/workflows/pr.yaml

@@ -127,6 +127,14 @@ jobs:
       cuda: '["12.2"]'
       extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY
       build_command: |
+        # For testing, will be removed when the package is released
+        git clone -b use-dfg https://github.com/KyleFromNVIDIA/rapids-build-backend.git
+        pushd rapids-build-backend
+        sed -e 's/^version =.*/version = "0.0.2"/' -i pyproject.toml
+        python -m pip install .
+        popd
+        export PIP_FIND_LINKS="file://${PWD}/rapids-build-backend"
+
         sccache -z;
         build-all --verbose;
         sccache -s;

.pre-commit-config.yaml

@@ -60,7 +60,7 @@ repos:
             pass_filenames: false
             language: python
     - repo: https://github.com/rapidsai/dependency-file-generator
-      rev: v1.8.0
+      rev: v1.11.0
       hooks:
           - id: rapids-dependency-file-generator
             args: ["--clean"]

build.sh

@@ -234,6 +234,13 @@ if (( ${CLEAN} == 1 )); then
 fi
 
 
+# For testing, will be removed when the package is released
+git clone -b use-dfg https://github.com/KyleFromNVIDIA/rapids-build-backend.git
+pushd rapids-build-backend
+sed -e 's/^version =.*/version = "0.0.2"/' -i pyproject.toml
+python -m pip install .
+popd
+
 ################################################################################
 # Configure for building all C++ targets
 if completeBuild || hasArg libcuml || hasArg prims || hasArg bench || hasArg prims-bench || hasArg cppdocs || hasArg cpp-mgtests; then

ci/build_python.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -13,15 +13,10 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-package_name="cuml"
-package_dir="python"
-
 version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
 export RAPIDS_PACKAGE_VERSION=${version}
 
 echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/_version.py"
 
 rapids-logger "Begin py build"
 

ci/build_wheel.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -10,46 +10,25 @@ source rapids-configure-sccache
 source rapids-date-string
 
 version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 # This is the version of the suffix with a preceding hyphen. It's used
 # everywhere except in the final wheel name.
 PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
 
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="${package_dir}/pyproject.toml"
-
-sed -i "/^name.*cuml/ s/= \"cuml\"/= \"cuml${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
 echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/_version.py"
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-for dep in cudf pylibraft raft-dask rmm; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-for dep in dask-cuda rapids-dask-dependency; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" ${pyproject_file}
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
 
 cd ${package_dir}
 
-SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" \
+# For testing, will be removed when the package is released
+git clone -b use-dfg https://github.com/KyleFromNVIDIA/rapids-build-backend.git
+pushd rapids-build-backend
+sed -e 's/^version =.*/version = "0.0.2"/' -i pyproject.toml
+python -m pip wheel . --no-deps
+popd
+
+PIP_FIND_LINKS="$PWD/rapids-build-backend" SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" \
   python -m pip wheel . \
     -w dist \
     -vvv \

conda/recipes/cuml-cpu/meta.yaml

@@ -29,6 +29,13 @@ requirements:
     - setuptools
     - scikit-build-core >=0.7.0
     - cython>=3.0.0
+    # TODO: Add rapids-build-backend once the package exists. Then we can remove dependencies up to the next TODO.
+    #- rapids-build-backend
+    - tomli
+    - tomli-w
+    - packaging
+    - rapids-dependency-file-generator
+    # TODO: Remove above dependencies after rapids-build-backend packages exist
   run:
     - python x.x
     - numpy>=1.23,<2.0a0

conda/recipes/cuml/meta.yaml

@@ -72,6 +72,13 @@ requirements:
     - scikit-build-core >=0.7.0
     - setuptools
     - treelite {{ treelite_version }}
+    # TODO: Add rapids-build-backend once the package exists. Then we can remove dependencies up to the next TODO.
+    #- rapids-build-backend
+    - tomli
+    - tomli-w
+    - packaging
+    - rapids-dependency-file-generator
+    # TODO: Remove above dependencies after rapids-build-backend packages exist
   run:
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
     {% if cuda_major == "11" %}

dependencies.yaml

@@ -13,6 +13,7 @@ files:
       - py_build
       - py_run
       - py_version
+      - rapids_build_backend
       - test_python
   cpp_all:
     output: conda
@@ -69,6 +70,13 @@ files:
     output: pyproject
     extras:
       table: build-system
+    includes:
+      - rapids_build_backend
+  py_rapids_build:
+    output: pyproject
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
     includes:
       - common_build
       - py_build
@@ -92,6 +100,17 @@ channels:
   - conda-forge
   - nvidia
 dependencies:
+  rapids_build_backend:
+    common:
+      - output_types: [pyproject] # TODO: Enable requirements once packages are available
+        packages:
+          - rapids-build-backend
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
+      - output_types: [conda]
+        packages:
+          - scikit-build-core>=0.7.0
   checks:
     common:
       - output_types: [conda, requirements]
@@ -159,54 +178,54 @@ dependencies:
         packages:
           - &pylibraft_conda pylibraft==24.6.*
           - &rmm_conda rmm==24.6.*
-          - scikit-build-core>=0.7.0
       - output_types: requirements
         packages:
           # pip recognizes the index as a global option for the requirements.txt file
           # This index is needed for cudf and rmm.
           - --extra-index-url=https://pypi.nvidia.com
           - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
-      - output_types: [pyproject, requirements]
-        packages:
-          - scikit-build-core[pyproject]>=0.7.0
     specific:
       - output_types: [conda, requirements, pyproject]
         matrices:
           - matrix:
               cuda: "12.*"
             packages:
               - cuda-python>=12.0,<13.0a0
-          - matrix: # All CUDA 11 versions
+          - matrix:
+              cuda: "11.*"
             packages:
               - cuda-python>=11.7.1,<12.0a0
+          - matrix:
+            packages:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}
             packages:
-              - pylibraft-cu12==24.6.*
-              - rmm-cu12==24.6.*
+              - pylibraft-cu12==24.6.*,>=0.0.0a0
+              - rmm-cu12==24.6.*,>=0.0.0a0
           - matrix: {cuda: "11.*"}
             packages:
-              - &pylibraft_cu11 pylibraft-cu11==24.6.*
-              - &rmm_cu11 rmm-cu11==24.6.*
-          - {matrix: null, packages: [*pylibraft_conda, *rmm_conda] }
+              - &pylibraft_cu11 pylibraft-cu11==24.6.*,>=0.0.0a0
+              - &rmm_cu11 rmm-cu11==24.6.*,>=0.0.0a0
+          - matrix:
+            packages:
 
   py_run:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - dask-cuda==24.6.*
           - joblib>=0.11
           - numba>=0.57
             # TODO: Is scipy really a hard dependency, or should
             # we make it optional (i.e. an extra for pip
             # installation/run_constrained for conda)?
           - scipy>=1.8.0
           - packaging
-          - rapids-dask-dependency==24.6.*
           - *treelite
       - output_types: conda
         packages:
+          - dask-cuda==24.6.*
+          - rapids-dask-dependency==24.6.*
           - &cudf_conda cudf==24.6.*
           - &cupy_conda cupy>=12.0.0
           - &dask_cudf_conda dask-cudf==24.6.*
@@ -217,33 +236,31 @@ dependencies:
           # This index is needed for cudf and rmm.
           - --extra-index-url=https://pypi.nvidia.com
           - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+      - output_types: [requirements, pyproject]
+        packages:
+          - dask-cuda==24.6.*,>=0.0.0a0
+          - rapids-dask-dependency==24.6.*,>=0.0.0a0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}
             packages:
-              - cudf-cu12==24.6.*
+              - cudf-cu12==24.6.*,>=0.0.0a0
               - cupy-cuda12x>=12.0.0
-              - dask-cudf-cu12==24.6.*
-              - pylibraft-cu12==24.6.*
-              - raft-dask-cu12==24.6.*
-              - rmm-cu12==24.6.*
+              - dask-cudf-cu12==24.6.*,>=0.0.0a0
+              - pylibraft-cu12==24.6.*,>=0.0.0a0
+              - raft-dask-cu12==24.6.*,>=0.0.0a0
+              - rmm-cu12==24.6.*,>=0.0.0a0
           - matrix: {cuda: "11.*"}
             packages: &py_run_packages_cu11
-              - cudf-cu11==24.6.*
-              - &cupy_pyproject_cu11 cupy-cuda11x>=12.0.0
-              - dask-cudf-cu11==24.6.*
+              - cudf-cu11==24.6.*,>=0.0.0a0
+              - &cupy_pyproject_cu11 cupy-cuda11x>=12.0.0,>=0.0.0a0
+              - dask-cudf-cu11==24.6.*,>=0.0.0a0
               - *pylibraft_cu11
-              - raft-dask-cu11==24.6.*
+              - raft-dask-cu11==24.6.*,>=0.0.0a0
               - *rmm_cu11
-          - matrix: null
+          - matrix:
             packages:
-              - *cudf_conda
-              - *cupy_pyproject_cu11
-              - *dask_cudf_conda
-              - *pylibraft_conda
-              - *raft_dask_conda
-              - *rmm_conda
   cuda_version:
     specific:
       - output_types: conda

python/cuml/_version.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,4 +18,7 @@
 __version__ = (
     importlib.resources.files("cuml").joinpath("VERSION").read_text().strip()
 )
-__git_commit__ = ""
+try:
+    __git_commit__ = importlib.resources.files("cuml").joinpath("GIT_COMMIT").read_text().strip()
+except FileNotFoundError:
+    __git_commit__ = ""

python/pyproject.toml

@@ -14,16 +14,10 @@
 
 [build-system]
 requires = [
-    "cmake>=3.26.4",
-    "cuda-python>=11.7.1,<12.0a0",
-    "cython>=3.0.0",
-    "ninja",
-    "pylibraft==24.6.*",
-    "rmm==24.6.*",
+    "rapids-build-backend",
     "scikit-build-core[pyproject]>=0.7.0",
-    "treelite==4.1.2",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
-build-backend = "scikit_build_core.build"
+build-backend = "rapids_build_backend.build"
 
 [tool.pytest.ini_options]
 markers = [
@@ -55,17 +49,11 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.9"
 dependencies = [
-    "cudf==24.6.*",
-    "cupy-cuda11x>=12.0.0",
-    "dask-cuda==24.6.*",
-    "dask-cudf==24.6.*",
+    "dask-cuda==24.6.*,>=0.0.0a0",
     "joblib>=0.11",
     "numba>=0.57",
     "packaging",
-    "pylibraft==24.6.*",
-    "raft-dask==24.6.*",
-    "rapids-dask-dependency==24.6.*",
-    "rmm==24.6.*",
+    "rapids-dask-dependency==24.6.*,>=0.0.0a0",
     "scipy>=1.8.0",
     "treelite==4.1.2",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -125,6 +113,17 @@ versioneer\.py |
 )/
 '''
 
+[tool.rapids-build-backend]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cmake>=3.26.4",
+    "cython>=3.0.0",
+    "ninja",
+    "treelite==4.1.2",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+commit-file = "cuml/GIT_COMMIT"
+dependencies-file = "../dependencies.yaml"
+
 [tool.scikit-build]
 build-dir = "build/{wheel_tag}"
 cmake.build-type = "Release"