From 08b7a46402675117dc335f922a32bccc6c05756d Mon Sep 17 00:00:00 2001 From: natsukium Date: Thu, 1 Jun 2023 00:33:31 +0900 Subject: [PATCH 1/3] catboost: init at 1.2.2 --- .../libraries/catboost/default.nix | 113 ++++++++++++++++++ .../libraries/catboost/remove-conan.patch | 34 ++++++ pkgs/top-level/all-packages.nix | 6 + 3 files changed, 153 insertions(+) create mode 100644 pkgs/development/libraries/catboost/default.nix create mode 100644 pkgs/development/libraries/catboost/remove-conan.patch diff --git a/pkgs/development/libraries/catboost/default.nix b/pkgs/development/libraries/catboost/default.nix new file mode 100644 index 0000000000000..713e705911bbf --- /dev/null +++ b/pkgs/development/libraries/catboost/default.nix @@ -0,0 +1,113 @@ +{ lib +, config +, stdenv +, fetchFromGitHub +, cmake +, libiconv +, llvmPackages +, ninja +, openssl +, python3Packages +, ragel +, yasm +, zlib +, cudaSupport ? config.cudaSupport +, cudaPackages ? {} +, pythonSupport ? false +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "catboost"; + version = "1.2.2"; + + src = fetchFromGitHub { + owner = "catboost"; + repo = "catboost"; + rev = "refs/tags/v${finalAttrs.version}"; + hash = "sha256-A1zCIqPOW21dHKBQHRtS+/sstZ2o6F8k71lmJFGn0+g="; + }; + + patches = [ + ./remove-conan.patch + ]; + + postPatch = '' + substituteInPlace cmake/common.cmake \ + --replace "\''${RAGEL_BIN}" "${ragel}/bin/ragel" \ + --replace "\''${YASM_BIN}" "${yasm}/bin/yasm" + + shopt -s globstar + for cmakelists in **/CMakeLists.*; do + sed -i "s/OpenSSL::OpenSSL/OpenSSL::SSL/g" $cmakelists + ${lib.optionalString (lib.versionOlder cudaPackages.cudaVersion "11.8") '' + sed -i 's/-gencode=arch=compute_89,code=sm_89//g' $cmakelists + sed -i 's/-gencode=arch=compute_90,code=sm_90//g' $cmakelists + ''} + done + ''; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + cmake + llvmPackages.bintools + ninja + (python3Packages.python.withPackages (ps: with ps; [ six ])) + ragel + yasm + ] ++ lib.optionals cudaSupport (with cudaPackages; [ + cuda_nvcc + ]); + + buildInputs = [ + openssl + zlib + ] ++ lib.optionals stdenv.isDarwin [ + libiconv + ] ++ lib.optionals cudaSupport (with cudaPackages; [ + cuda_cudart + cuda_cccl + libcublas + ]); + + env = { + CUDAHOSTCXX = lib.optionalString cudaSupport "${stdenv.cc}/bin/cc"; + NIX_CFLAGS_LINK = lib.optionalString stdenv.isLinux "-fuse-ld=lld"; + NIX_LDFLAGS = "-lc -lm"; + }; + + cmakeFlags = [ + "-DCMAKE_BINARY_DIR=$out" + "-DCMAKE_POSITION_INDEPENDENT_CODE=on" + "-DCATBOOST_COMPONENTS=app;libs${lib.optionalString pythonSupport ";python-package"}" + ] ++ lib.optionals cudaSupport [ + "-DHAVE_CUDA=on" + ]; + + installPhase = '' + runHook preInstall + + mkdir $dev + cp -r catboost $dev + install -Dm555 catboost/app/catboost -t $out/bin + install -Dm444 catboost/libs/model_interface/static/lib/libmodel_interface-static-lib.a -t $out/lib + install -Dm444 catboost/libs/model_interface/libcatboostmodel${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib + install -Dm444 catboost/libs/train_interface/libcatboost${stdenv.hostPlatform.extensions.sharedLibrary} -t $out/lib + + runHook postInstall + ''; + + meta = with lib; { + description = "High-performance library for gradient boosting on decision trees"; + longDescription = '' + A fast, scalable, high performance Gradient Boosting on Decision Trees + library, used for ranking, classification, regression and other machine + learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU. + ''; + license = licenses.asl20; + platforms = platforms.unix; + homepage = "https://catboost.ai"; + maintainers = with maintainers; [ PlushBeaver ]; + mainProgram = "catboost"; + }; +}) diff --git a/pkgs/development/libraries/catboost/remove-conan.patch b/pkgs/development/libraries/catboost/remove-conan.patch new file mode 100644 index 0000000000000..6f96b7989a587 --- /dev/null +++ b/pkgs/development/libraries/catboost/remove-conan.patch @@ -0,0 +1,34 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index becd2ad03c..7e3c8c99b1 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -27,7 +27,6 @@ cmake_policy(SET CMP0104 OLD) + + include(cmake/archive.cmake) + include(cmake/common.cmake) +-include(cmake/conan.cmake) + include(cmake/cuda.cmake) + include(cmake/cython.cmake) + include(cmake/fbs.cmake) +@@ -37,21 +36,6 @@ include(cmake/recursive_library.cmake) + include(cmake/swig.cmake) + include(cmake/global_vars.cmake) + +-if (CMAKE_CROSSCOMPILING) +- include(${CMAKE_BINARY_DIR}/conan_paths.cmake) +-else() +- conan_cmake_autodetect(settings) +- conan_cmake_install( +- PATH_OR_REFERENCE ${CMAKE_SOURCE_DIR} +- INSTALL_FOLDER ${CMAKE_BINARY_DIR} +- BUILD missing +- REMOTE conancenter +- SETTINGS ${settings} +- ENV "CONAN_CMAKE_GENERATOR=${CMAKE_GENERATOR}" +- CONF "tools.cmake.cmaketoolchain:generator=${CMAKE_GENERATOR}" +- ) +-endif() +- + if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) + elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND HAVE_CUDA) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index eb2c97d720fb6..f44b03954477f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -20899,6 +20899,12 @@ with pkgs; captive-browser = callPackage ../applications/networking/browsers/captive-browser { }; + catboost = callPackage ../development/libraries/catboost { + # catboost requires clang 12+ for build + # after bumping the default version of llvm, check for compatibility with the cuda backend and pin it. + inherit (llvmPackages_12) stdenv; + }; + ndn-cxx = callPackage ../development/libraries/ndn-cxx { }; ndn-tools = callPackage ../tools/networking/ndn-tools { }; From fc84d025c6c6823b28800fda29ac65f8975efa67 Mon Sep 17 00:00:00 2001 From: natsukium Date: Thu, 1 Jun 2023 00:34:55 +0900 Subject: [PATCH 2/3] python310Packages.catboost: 1.0.5 -> 1.2.2 Diff: https://github.com/catboost/catboost/compare/v1.0.5...v1.2.2 Changelog: https://github.com/catboost/catboost/releases/tag/v1.2.2 --- .../python-modules/catboost/default.nix | 96 +++++----- .../python-modules/catboost/nix-support.patch | 173 ------------------ pkgs/top-level/python-packages.nix | 7 +- 3 files changed, 47 insertions(+), 229 deletions(-) delete mode 100644 pkgs/development/python-modules/catboost/nix-support.patch diff --git a/pkgs/development/python-modules/catboost/default.nix b/pkgs/development/python-modules/catboost/default.nix index 1939b7c13a6d0..30e140a831428 100644 --- a/pkgs/development/python-modules/catboost/default.nix +++ b/pkgs/development/python-modules/catboost/default.nix @@ -1,64 +1,50 @@ -{ buildPythonPackage, fetchFromGitHub, lib, pythonOlder -, clang_12, python -, graphviz, matplotlib, numpy, pandas, plotly, scipy, six -, withCuda ? false, cudatoolkit }: - -buildPythonPackage rec { - pname = "catboost"; - # nixpkgs-update: no auto update - version = "1.0.5"; - - disabled = pythonOlder "3.4"; +{ lib +, buildPythonPackage +, catboost +, python +, graphviz +, matplotlib +, numpy +, pandas +, plotly +, scipy +, setuptools +, six +, wheel +}: + +buildPythonPackage { + inherit (catboost) pname version src meta; + format = "pyproject"; + + sourceRoot = "source/catboost/python-package"; + + nativeBuildInputs = [ + setuptools + wheel + ]; - src = fetchFromGitHub { - owner = "catboost"; - repo = "catboost"; - rev = "refs/tags/v${version}"; - hash = "sha256-ILemeZUBI9jPb9G6F7QX/T1HaVhQ+g6y7YmsT6DFCJk"; - }; + propagatedBuildInputs = [ + graphviz + matplotlib + numpy + pandas + plotly + scipy + six + ]; - nativeBuildInputs = [ clang_12 ]; + buildPhase = '' + runHook preBuild - propagatedBuildInputs = [ graphviz matplotlib numpy pandas scipy plotly six ] - ++ lib.optionals withCuda [ cudatoolkit ]; + # these arguments must set after bdist_wheel + ${python.pythonForBuild.interpreter} setup.py bdist_wheel --no-widget --prebuilt-extensions-build-root-dir=${lib.getDev catboost} - patches = [ - ./nix-support.patch - ]; - - postPatch = '' - # substituteInPlace is too slow for these large files, and the target has lots of numbers in it that change often. - sed -e 's|\$(YMAKE_PYTHON3-.*)/python3|${python.interpreter}|' -i make/*.makefile + runHook postBuild ''; - preBuild = '' - cd catboost/python-package - ''; - setupPyBuildFlags = [ "--with-ymake=no" ]; - CUDA_ROOT = lib.optional withCuda cudatoolkit; - enableParallelBuilding = true; + # setup a test is difficult + doCheck = false; - # Tests use custom "ya" tool, not yet supported. - dontUseSetuptoolsCheck = true; pythonImportsCheck = [ "catboost" ]; - - passthru = { - # Do not update to catboost 1.1.x because the patch doesn't apply cleanly - skipBulkUpdate = true; - }; - - meta = with lib; { - description = "High-performance library for gradient boosting on decision trees."; - longDescription = '' - A fast, scalable, high performance Gradient Boosting on Decision Trees - library, used for ranking, classification, regression and other machine - learning tasks for Python, R, Java, C++. Supports computation on CPU and GPU. - ''; - license = licenses.asl20; - platforms = [ "x86_64-linux" ]; - homepage = "https://catboost.ai"; - maintainers = with maintainers; [ PlushBeaver ]; - # _catboost.pyx.cpp:226822:19: error: use of undeclared identifier '_PyGen_Send' - broken = withCuda; - }; } diff --git a/pkgs/development/python-modules/catboost/nix-support.patch b/pkgs/development/python-modules/catboost/nix-support.patch deleted file mode 100644 index b8294420e09c3..0000000000000 --- a/pkgs/development/python-modules/catboost/nix-support.patch +++ /dev/null @@ -1,173 +0,0 @@ -diff --git a/catboost/python-package/setup.py b/catboost/python-package/setup.py -index fe9251a21f..86b880c5d0 100644 ---- a/catboost/python-package/setup.py -+++ b/catboost/python-package/setup.py -@@ -80,7 +80,7 @@ class Helper(object): - self.with_cuda = os.environ.get('CUDA_PATH') or os.environ.get('CUDA_ROOT') or None - self.os_sdk = 'local' - self.with_ymake = True -- self.parallel = None -+ self.parallel = os.environ.get('NIX_BUILD_CORES') or None - - def finalize_options(self): - if os.path.exists(str(self.with_cuda)): -@@ -222,11 +222,12 @@ class build_ext(_build_ext): - - def build_with_make(self, topsrc_dir, build_dir, catboost_ext, put_dir, verbose, dry_run): - logging.info('Buildling {} with gnu make'.format(catboost_ext)) -- makefile = 'python{}.{}CLANG11-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '') -+ makefile = 'python{}.{}CLANG12-LINUX-X86_64.makefile'.format(python_version()[0], 'CUDA.' if self.with_cuda else '') - make_cmd = [ - 'make', '-f', '../../make/' + makefile, -- 'CC=clang-11', -- 'CXX=clang++-11', -+ 'CC=clang', -+ 'CXX=clang++', -+ 'PYTHON=python{}'.format(python_version()[0]), - 'BUILD_ROOT=' + build_dir, - 'SOURCE_ROOT=' + topsrc_dir, - ] -diff --git a/make/python2.CLANG12-LINUX-X86_64.makefile b/make/python2.CLANG12-LINUX-X86_64.makefile -index b49a36fb3f..33996af995 100644 ---- a/make/python2.CLANG12-LINUX-X86_64.makefile -+++ b/make/python2.CLANG12-LINUX-X86_64.makefile -@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd) - SOURCE_ROOT = $(shell pwd) - PYTHON = $(shell which python) - --ifneq ($(MAKECMDGOALS),help) --define _CC_TEST --__clang_major__ __clang_minor__ --endef -- --_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -) --$(info _CC_VERSION = '$(_CC_VERSION)') -- --ifneq '$(_CC_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif -- --ifneq ($(MAKECMDGOALS),help) --define _CXX_TEST --__clang_major__ __clang_minor__ --endef -- --_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -) --$(info _CXX_VERSION = '$(_CXX_VERSION)') -- --ifneq '$(_CXX_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif - - - all\ -diff --git a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile -index 82935b297e..093cc86532 100644 ---- a/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile -+++ b/make/python2.CUDA.CLANG12-LINUX-X86_64.makefile -@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd) - SOURCE_ROOT = $(shell pwd) - PYTHON = $(shell which python) - --ifneq ($(MAKECMDGOALS),help) --define _CC_TEST --__clang_major__ __clang_minor__ --endef -- --_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -) --$(info _CC_VERSION = '$(_CC_VERSION)') -- --ifneq '$(_CC_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif -- --ifneq ($(MAKECMDGOALS),help) --define _CXX_TEST --__clang_major__ __clang_minor__ --endef -- --_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -) --$(info _CXX_VERSION = '$(_CXX_VERSION)') -- --ifneq '$(_CXX_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif - - - all\ -diff --git a/make/python3.CLANG12-LINUX-X86_64.makefile b/make/python3.CLANG12-LINUX-X86_64.makefile -index 1c5d646ae4..6c091fbe17 100644 ---- a/make/python3.CLANG12-LINUX-X86_64.makefile -+++ b/make/python3.CLANG12-LINUX-X86_64.makefile -@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd) - SOURCE_ROOT = $(shell pwd) - PYTHON = $(shell which python) - --ifneq ($(MAKECMDGOALS),help) --define _CC_TEST --__clang_major__ __clang_minor__ --endef -- --_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -) --$(info _CC_VERSION = '$(_CC_VERSION)') -- --ifneq '$(_CC_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif -- --ifneq ($(MAKECMDGOALS),help) --define _CXX_TEST --__clang_major__ __clang_minor__ --endef -- --_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -) --$(info _CXX_VERSION = '$(_CXX_VERSION)') -- --ifneq '$(_CXX_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif - - - all\ -diff --git a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile -index fcdb75a719..4e1dbc3cd7 100644 ---- a/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile -+++ b/make/python3.CUDA.CLANG12-LINUX-X86_64.makefile -@@ -4,31 +4,6 @@ BUILD_ROOT = $(shell pwd) - SOURCE_ROOT = $(shell pwd) - PYTHON = $(shell which python) - --ifneq ($(MAKECMDGOALS),help) --define _CC_TEST --__clang_major__ __clang_minor__ --endef -- --_CC_VERSION = $(shell echo '$(_CC_TEST)' | $(CC) -E -P -) --$(info _CC_VERSION = '$(_CC_VERSION)') -- --ifneq '$(_CC_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif -- --ifneq ($(MAKECMDGOALS),help) --define _CXX_TEST --__clang_major__ __clang_minor__ --endef -- --_CXX_VERSION = $(shell echo '$(_CXX_TEST)' | $(CXX) -E -P -) --$(info _CXX_VERSION = '$(_CXX_VERSION)') -- --ifneq '$(_CXX_VERSION)' '12 0' -- $(error clang 12.0 is required) --endif --endif - - - all\ diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index df54f262c346f..1554c30425da7 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1769,7 +1769,12 @@ self: super: with self; { catalogue = callPackage ../development/python-modules/catalogue { }; - catboost = callPackage ../development/python-modules/catboost { }; + catboost = callPackage ../development/python-modules/catboost { + catboost = pkgs.catboost.override { + pythonSupport = true; + python3Packages = self; + }; + }; catppuccin = callPackage ../development/python-modules/catppuccin { }; From 0b93e56237687efe526ab2a595356debee9da50c Mon Sep 17 00:00:00 2001 From: natsukium Date: Sat, 24 Jun 2023 13:54:01 +0900 Subject: [PATCH 3/3] catboost: add natsukium to maintainers --- pkgs/development/libraries/catboost/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/libraries/catboost/default.nix b/pkgs/development/libraries/catboost/default.nix index 713e705911bbf..fc18eef2ca893 100644 --- a/pkgs/development/libraries/catboost/default.nix +++ b/pkgs/development/libraries/catboost/default.nix @@ -107,7 +107,7 @@ stdenv.mkDerivation (finalAttrs: { license = licenses.asl20; platforms = platforms.unix; homepage = "https://catboost.ai"; - maintainers = with maintainers; [ PlushBeaver ]; + maintainers = with maintainers; [ PlushBeaver natsukium ]; mainProgram = "catboost"; }; })